diff --git a/.gitignore b/.gitignore index 007c370..cb8459d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ local.properties /captures .externalNativeBuild .cxx + +scratch/ diff --git a/docs/Handling Obscured Notifications.md b/docs/Handling Obscured Notifications.md new file mode 100644 index 0000000..0b1adad --- /dev/null +++ b/docs/Handling Obscured Notifications.md @@ -0,0 +1,124 @@ +# Handling Obscured Notifications + +## What is an "obscured" notification? + +An **obscured notification** is one whose content is absent from the +[`NotificationListenerService`](https://developer.android.com/reference/android/service/notification/NotificationListenerService) +(NLS) payload — no title, text, or extras — but whose content IS visible in the notification +shade and IS readable via the +[`AccessibilityService`](https://developer.android.com/reference/android/accessibilityservice/AccessibilityService) +accessibility tree. + +This happens when an app posts a `GROUP_SUMMARY` notification that carries only metadata +(flags, groupKey, etc.) and relies on the rendered shade row — not the NLS extras — to display +content to the user. + +--- + +## Two sub-classes of obscured notification + +### 1. Stale-obscured (confirmed, common) + +**Behavior during live delivery (app running):** +The app posts a silent `GROUP_SUMMARY` *immediately followed* by one or more content-bearing +child notifications (e.g. `CHAT_CHIME`, `MessagingStyle`) within milliseconds. NLS sees both. +The 300 ms `PENDING_LOOKUP_DELAY_MS` cancellation window in `schedulePendingLookup` gives the +content-bearing child time to arrive and cancel the accessibility lookup before it fires. + +Result: **the normal NLS path speaks the content; the accessibility tree is not consulted for the +content itself.** Note: `MyAccessibilityService` must still be enabled — it is a hard requirement +that gates all NLS processing, including live notifications (see +[Requirement.ACCESSIBILITY_SERVICE] in `StartupCoordinator.kt`). + +**Behavior during stale catch-up (app just started):** +When the app restarts and `initializeActiveNotifications` calls `getActiveNotifications()`, the +content-bearing child notifications may already be gone — consumed, dismissed, or never +re-posted. Only the empty `GROUP_SUMMARY` remains. No cancel arrives within 300 ms, so the +accessibility lookup fires: the shade is opened, collapsed rows are expanded, and the row +content is read from the accessibility tree. + +**Confirmed stale-obscured packages:** +- `com.google.android.apps.dynamite` — Google Chat. Live delivery: GROUP_SUMMARY + CHAT_CHIME + sibling. Stale catch-up: GROUP_SUMMARY only. Accessibility is the sole source of content at + catch-up time. + +### 2. Always-obscured (theoretical, unconfirmed) + +**Hypothesis:** some apps never post a content-bearing sibling notification — not even during +live delivery. The GROUP_SUMMARY is the only notification they ever post for a given message. +Content is only available in the shade. These packages would require the accessibility path for +*both* live and stale delivery, and the `PENDING_LOOKUP_DELAY_MS` cancellation window would +never fire regardless of whether the app was just started or has been running for hours. + +**Theory on which apps this applies to:** if this category exists, it is almost certainly +limited to **system-signed / AOSP / Google apps**. Third-party apps generally provide readable +content in the NLS payload or in a sibling notification. System apps often have parallel +delivery channels (e.g. FCM data messages delivered directly to the app process) and may use +the `GROUP_SUMMARY` notification solely as a visual shade badge. + +**Currently uncharacterized packages** (appeared in logs, live vs. stale behavior not yet +confirmed): +- `com.google.android.googlequicksearchbox` — Google search / assistant + +--- + +## Detection and corpus-building + +`ObscuredNotificationLogger` records every accessibility lookup outcome +(`FOUND` / `NOT_FOUND` / `ACCESSIBILITY_UNAVAILABLE`) keyed by `packageName`. + +To identify candidates for the **always-obscured** category, look for packages that +consistently reach the accessibility path even during live notification delivery — not only at +app startup. A `packageName` that triggers `findRowForAppLabel` on freshly arriving +notifications (and consistently returns `FOUND`) is a strong candidate. + +--- + +## Architecture summary + +``` +onNotificationPosted(sbn) + └─ defaultOnNotificationPosted → ParsedIgnored + └─ schedulePendingLookup(300 ms) + │ + ├─ [live, stale-obscured] sibling child arrives within 300 ms + │ └─ cancelPendingLookup → spoken via NLS path + │ + └─ [stale catch-up, or always-obscured live] 300 ms elapses with no cancel + └─ MyAccessibilityService.findRowForAppLabel(appLabel) + └─ ShadeRowSearchQueue: open shade → expand rows → match appName + └─ callback → speakShadeRows → TTS +``` + +Multiple concurrent `findRowForAppLabel` calls (e.g. Chat and Google both firing within 30 ms +at startup) are serialized by `ShadeRowSearchQueue` — each search waits for the previous to +finish before the shade is handed to it. + +--- + +## Key constants + +| Constant | Location | Value | Purpose | +|---|---|---|---| +| `PENDING_LOOKUP_DELAY_MS` | `MyNotificationListenerService` | 300 ms | Window for content-bearing child to cancel the accessibility lookup | +| `ShadeDelays.FAST.shadeSettle` | `ShadeDelays` | 600 ms | Shade animation settle time (production) | +| `ShadeDelays.SLOW.shadeSettle` | `ShadeDelays` | 3000 ms | Shade animation settle time (debug/observation) | +| `ShadeDelays.FAST.preExpand` | `ShadeDelays` | 0 ms | Pause before each row expansion (production) | +| `ShadeDelays.SLOW.preExpand` | `ShadeDelays` | 2000 ms | Pause before each row expansion (debug/observation) | +| `MAX_SCROLL_ATTEMPTS` | `ShadeRowSearchQueue` | 10 | Max scroll attempts before giving up on off-screen rows | + +Toggle `MyAccessibilityService.DEBUG_SLOW_MODE = true` to use the SLOW delays for visual +step-by-step observation of the shade automation. Set false and confirm FAST values once +minimum working delays are established. + +--- + +## Known limitations + +- **Multi-account apps:** if two accounts of the same app post GROUP_SUMMARY simultaneously, + the first matching shade row wins; the second is not spoken. +- **Screen locked:** when the screen is locked, expand/collapse behavior via the accessibility + tree is unknown and the shade row content may be unavailable. +- **`com.google.android.gm` (Gmail):** sends `tickerText` with partial content via NLS; the + accessibility path is not currently required but a bespoke `NotificationParser` may improve + the spoken output. diff --git a/mobile/README.md b/mobile/README.md index a087668..d47208e 100644 --- a/mobile/README.md +++ b/mobile/README.md @@ -1,9 +1,22 @@ -## Architecture: +## Architecture + +### Startup / permissions - Single Activity + Compose UI gate (no navigation trampoline) - System SplashScreen API (androidx.core.splashscreen) -- StartupCoordinator (ViewModel) as single source of truth -- Requirement enum → hard gates (blocks isReady) -- Advisory enum → soft hints (never blocks isReady) -- ListenerEnabledMonitor → callbackFlow/ContentObserver for instant mid-session revocation detection (no ON_RESUME polling needed for NOTIFICATION_LISTENER specifically) +- `StartupCoordinator` (ViewModel) as single source of truth +- `Requirement` enum → hard gates (blocks `isReady`) +- `Advisory` enum → soft hints (never blocks `isReady`) +- `ListenerEnabledMonitor` → callbackFlow/ContentObserver for instant mid-session revocation detection (no ON_RESUME polling needed for NOTIFICATION_LISTENER specifically) - DisposableEffect + ON_RESUME still re-checks POST_NOTIFICATIONS and BATTERY_OPTIMIZATION which have no observable Settings key + +### Notification delivery +Two paths — see [`notification/parsers/README.md`](src/main/java/llc/lookatwhataicando/notifai/notification/parsers/README.md) for full ground truth. + +- **`MyNotificationListenerService`** — primary path; handles all live notifications via `onNotificationPosted` + per-package `NotificationParser` subclasses +- **`MyAccessibilityService`** — fallback for "obscured" notifications (empty `GROUP_SUMMARY` payload, content only in the shade); **primarily needed at launch** to catch up on notifications that arrived before the app started; largely not needed for live delivery + +### Accessibility permission — true scope +The `ACCESSIBILITY_SERVICE` requirement is a hard gate in `StartupCoordinator`, but its real +impact is narrow: **without it, only the launch catch-up path is degraded**. Live notifications +are unaffected. See `StartupCoordinator.Requirement.ACCESSIBILITY_SERVICE` KDoc for details. diff --git a/mobile/src/androidTest/java/llc/lookatwhataicando/notifai/ExampleInstrumentedTest.kt b/mobile/src/androidTest/java/llc/lookatwhataicando/notifai/ExampleInstrumentedTest.kt index 6b87aa4..f2e073f 100644 --- a/mobile/src/androidTest/java/llc/lookatwhataicando/notifai/ExampleInstrumentedTest.kt +++ b/mobile/src/androidTest/java/llc/lookatwhataicando/notifai/ExampleInstrumentedTest.kt @@ -1,13 +1,11 @@ package llc.lookatwhataicando.notifai -import androidx.test.platform.app.InstrumentationRegistry import androidx.test.ext.junit.runners.AndroidJUnit4 - +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals import org.junit.Test import org.junit.runner.RunWith -import org.junit.Assert.* - /** * Instrumented test, which will execute on an Android device. * diff --git a/mobile/src/main/AndroidManifest.xml b/mobile/src/main/AndroidManifest.xml index 1d244fe..a74a141 100644 --- a/mobile/src/main/AndroidManifest.xml +++ b/mobile/src/main/AndroidManifest.xml @@ -79,6 +79,19 @@ android:resource="@xml/notification_listener_service_meta" /> + + + + + + + List?, + private val getLastSnapshot: () -> List, + private val openShade: () -> Unit, + private val closeShade: () -> Unit, +) { + companion object { + private val TAG = FooLog.TAG(DebugShadeScan::class) + private const val EXPAND_SETTLE_MS = 1000L + private const val SCROLL_SETTLE_MS = 500L + private const val MAX_SCROLL_ATTEMPTS = 10 + private const val MAX_EXPAND_PASSES_PER_SCROLL = 20 + } + + private val mainHandler = Handler(Looper.getMainLooper()) + private var active = false + + fun debugFullShadeScan() { + if (active) { + FooLog.d(TAG, "debugFullShadeScan: scan already in progress, ignoring") + return + } + active = true + FooLog.i(TAG, "debugFullShadeScan: starting — opening shade") + openShade() + mainHandler.postDelayed( + { debugScanPass(linkedSetOf(), MAX_SCROLL_ATTEMPTS, MAX_EXPAND_PASSES_PER_SCROLL) }, + delays.shadeSettle, + ) + } + + /** + * Expands collapsed rows one at a time, each separated by [EXPAND_SETTLE_MS]. + * + * Each call scans all visible rows via actionList: + * - Already expanded (ACTION_COLLAPSE present): skip. + * - Collapsed (ACTION_EXPAND present): expand row, schedule next pass after [EXPAND_SETTLE_MS], + * then RETURN — only one expansion per delay interval. + * - Neither action: skip. + * + * Once no collapsed row is found, proceeds to [debugScanCollect]. + * Guarded by [expandPassesLeft] to prevent an unbounded loop. + */ + private fun debugScanPass( + accumulated: LinkedHashSet, + scrollAttemptsLeft: Int, + expandPassesLeft: Int, + ) { + val rows = NotificationShadeSnapshot.getLiveRowNodes(getWindows()) + FooLog.v(TAG, "debugScanPass: ${rows.size} rows visible (expandPassesLeft=$expandPassesLeft)") + rows.forEachIndexed { _, rowNode -> + if (rowNode.actionList.any { it.id == AccessibilityNodeInfo.ACTION_COLLAPSE }) return@forEachIndexed + if (rowNode.actionList.any { it.id == AccessibilityNodeInfo.ACTION_EXPAND }) { + rowNode.performAction(AccessibilityNodeInfo.ACTION_EXPAND) + if (expandPassesLeft > 0) { + mainHandler.postDelayed( + { debugScanPass(accumulated, scrollAttemptsLeft, expandPassesLeft - 1) }, + EXPAND_SETTLE_MS, + ) + } else { + FooLog.w(TAG, "debugScanPass: expand pass limit reached") + debugScanCollect(accumulated, scrollAttemptsLeft) + } + return // only one expansion per delay interval + } + } + debugScanCollect(accumulated, scrollAttemptsLeft) + } + + /** Snapshot current rows, merge into [accumulated], then attempt a scroll. */ + private fun debugScanCollect(accumulated: LinkedHashSet, scrollAttemptsLeft: Int) { + val snapshot = getLastSnapshot() + val sb = StringBuilder("debugScanCollect: ${snapshot.size} rows in snapshot:") + snapshot.forEachIndexed { i, row -> sb.append("\n [$i] $row") } + FooLog.i(TAG, sb.toString()) + accumulated.addAll(snapshot.filter { !it.isEmpty }) + val rowCountBeforeScroll = NotificationShadeSnapshot.getLiveRowNodes(getWindows()).size + debugScanScroll(accumulated, scrollAttemptsLeft, rowCountBeforeScroll) + } + + /** + * Attempt [AccessibilityNodeInfo.ACTION_SCROLL_FORWARD] on the container. After a short + * settle, compare row count: if new rows appeared, continue scanning; otherwise finish. + */ + private fun debugScanScroll( + accumulated: LinkedHashSet, + scrollAttemptsLeft: Int, + rowCountBefore: Int, + ) { + if (scrollAttemptsLeft <= 0) { + FooLog.i(TAG, "debugScanScroll: scroll attempts exhausted") + debugScanFinish(accumulated) + return + } + val container = NotificationShadeSnapshot.getLiveContainerNode(getWindows()) + if (container == null) { + FooLog.w(TAG, "debugScanScroll: container not found") + debugScanFinish(accumulated) + return + } + val scrolled = container.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD) + FooLog.v(TAG, "debugScanScroll: ACTION_SCROLL_FORWARD=$scrolled ($scrollAttemptsLeft attempts left)") + if (!scrolled) { + FooLog.i(TAG, "debugScanScroll: scroll returned false — at bottom or scroll unsupported") + debugScanFinish(accumulated) + return + } + mainHandler.postDelayed({ + val rowCountAfter = NotificationShadeSnapshot.getLiveRowNodes(getWindows()).size + if (rowCountAfter <= rowCountBefore) { + FooLog.i(TAG, "debugScanScroll: no new rows after scroll ($rowCountBefore → $rowCountAfter) — at bottom") + debugScanFinish(accumulated) + } else { + FooLog.v(TAG, "debugScanScroll: new rows after scroll ($rowCountBefore → $rowCountAfter) — continuing") + debugScanPass(accumulated, scrollAttemptsLeft - 1, MAX_EXPAND_PASSES_PER_SCROLL) + } + }, SCROLL_SETTLE_MS) + } + + /** Log the deduplicated corpus then close the shade after the observation delay. */ + private fun debugScanFinish(accumulated: LinkedHashSet) { + val rows = accumulated.toList() + val sb = StringBuilder("debugScanFinish: complete corpus (${rows.size} rows):") + rows.forEachIndexed { i, row -> sb.append("\n [$i] $row") } + FooLog.i(TAG, sb.toString()) + mainHandler.postDelayed({ + FooLog.i(TAG, "debugScanFinish: closing shade") + closeShade() + active = false + }, delays.preClose) + } +} diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/MainActivity.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/MainActivity.kt index 15ec3b2..820e0b3 100644 --- a/mobile/src/main/java/llc/lookatwhataicando/notifai/MainActivity.kt +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/MainActivity.kt @@ -299,6 +299,23 @@ fun PermissionsGateScreen( onClick = { FooNotification.startActivityNotificationListenerSettings(context) } ) else null ) + Spacer(Modifier.height(12.dp)) + // ── Required ACCESSIBILITY_SERVICE card ────────────────────── + val accessibilityMissing = Requirement.ACCESSIBILITY_SERVICE in snapshot.missing + RequirementCard( + icon = Icons.Outlined.Lock, + title = "Accessibility Service", + description = "Required for all notification processing. Without this, no " + + "notifications are spoken — not live arrivals, nor launch catch-up. " + + "Also enables reading content from apps like Google Chat that don't " + + "expose it via standard APIs. " + + "Enable `NotifAI` in the Accessibility settings that open.", + isMissing = accessibilityMissing, + primaryAction = if (accessibilityMissing) PermissionAction( + label = "Enable NotifAI Accessibility", + onClick = { FooPlatformUtils.showAccessibilitySettings(context) } + ) else null + ) // ── Recommended BATTERY_OPTIMIZATION card ──────────────────── if (forceShowAdvisories || Advisory.BATTERY_OPTIMIZATION in snapshot.advisories) { Spacer(Modifier.height(12.dp)) diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/MyAccessibilityService.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/MyAccessibilityService.kt new file mode 100644 index 0000000..390ad7f --- /dev/null +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/MyAccessibilityService.kt @@ -0,0 +1,235 @@ +package llc.lookatwhataicando.notifai + +import android.accessibilityservice.AccessibilityService +import android.annotation.SuppressLint +import android.view.accessibility.AccessibilityEvent +import androidx.annotation.RequiresApi +import com.smartfoo.android.core.logging.FooLog + +/** + * ## Behavioral ground truth — when and why this service is needed + * + * ### Live notifications (app already running) + * [MyNotificationListenerService] (NLS) handles these entirely. Apps such as Google Chat post + * a silent `GROUP_SUMMARY` notification (no title/text/extras) immediately followed by one or + * more content-bearing child notifications within milliseconds. NLS sees both; the 300 ms + * cancellation window in `MyNotificationListenerService.schedulePendingLookup` lets the + * content-bearing child cancel any pending accessibility lookup before it fires. Result: content + * is spoken via the normal NLS path with **no accessibility service involvement**. + * + * ### Launch / catch-up (app just started) — primary scenario + * When the app launches and iterates `getActiveNotifications()`, the content-bearing child + * notifications may already be gone — consumed/dismissed after the previous app instance + * processed them, or simply never re-posted. Only the empty `GROUP_SUMMARY` remains. NLS cannot + * read content from it. **This is the primary scenario where `MyAccessibilityService` is + * required:** it opens the notification shade, expands any collapsed rows, and reads their + * content via the accessibility tree so the app can speak notifications it missed. + * + * See [llc.lookatwhataicando.notifai.notification.ObscuredNotification] for the confirmed + * package `com.google.android.apps.dynamite` (Google Chat) and the stale-obscured model. + * + * ### Always-obscured live notifications — theoretical, unconfirmed + * Some apps may never post a content-bearing sibling notification — not even during live + * delivery. For these packages the accessibility path would fire unconditionally (both live + * and stale). If this category exists, it is theorized to be limited to system-signed / AOSP / + * Google apps. See `notification/parsers/README.md` for detection guidance. + * + * ### Secondary capability + * Provides global navigation actions (open/dismiss shade, back, home, screenshot, media + * play/pause) that NLS does not expose. + * + * ### Practical impact without this permission + * Live notifications from stale-obscured apps work correctly (NLS reads their content-bearing + * sibling). Only the launch catch-up path is degraded — the app silently skips stale obscured + * notifications whose content-bearing child is already gone. If any always-obscured apps exist, + * their live notifications would also be lost without this permission. + * + * --- + * + * ### Developer note — Android Studio kills Accessibility on deploy + * The app's Accessibility permission is disabled whenever it is launched or stopped from + * Android Studio. To keep Accessibility enabled across a deploy: + * ``` + * adb shell am start -n llc.lookatwhataicando.notifai/.MainActivity + * ``` + * Or re-enable Accessibility via the permissions check dialog that appears at launch. + * + * NOTE: `@SuppressLint("AccessibilityPolicy")` suppresses the following lint warning: + * "AccessibilityService API usage must align with the API policy guidelines. + * It must not be used to bypass privacy controls or change settings without consent." + * This code honors that. + */ +@SuppressLint("AccessibilityPolicy") +class MyAccessibilityService : AccessibilityService() { + companion object { + var instance: MyAccessibilityService? = null + private set + + private val TAG = FooLog.TAG(MyAccessibilityService::class) + + private val VERBOSE_LOG_ROOT_NULL = false + private val VERBOSE_LOG_SHADE_EMPTY = false + + /** + * Slows the accessibility state machine for step-by-step visual observation. + * + * When true, [ShadeDelays.SLOW] is used so each expand/scroll/close step is visually + * distinct. Set false and confirm [ShadeDelays.FAST] values once minimum working delays + * are known. + */ + @Suppress("SimplifyBooleanWithConstants", "KotlinConstantConditions") + var DEBUG_SLOW_MODE = false && BuildConfig.DEBUG + val delays: ShadeDelays = if (DEBUG_SLOW_MODE) ShadeDelays.SLOW else ShadeDelays.FAST + + /** + * When true, any [MyNotificationListenerService] ParsedIgnored notification triggers a + * full top-to-bottom shade scan ([DebugShadeScan]) instead of the normal app-label + * search. No TTS is spoken. Disable when switching back to the normal search path. + */ + var DEBUG_FULL_SCAN_MODE = false + + private val ACCESSIBILITY_EVENTS_NOTIFICATION_RELEVANT = setOf( + AccessibilityEvent.TYPE_NOTIFICATION_STATE_CHANGED, + AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED, + AccessibilityEvent.TYPE_WINDOWS_CHANGED, + ) + } + + // ------------------------------------------------------------------------- + // Global action helpers + // ------------------------------------------------------------------------- + + fun actionBack() { + FooLog.i(TAG, "actionBack()") + performGlobalAction(GLOBAL_ACTION_BACK) + } + + fun actionHome() { + FooLog.i(TAG, "actionHome()") + performGlobalAction(GLOBAL_ACTION_HOME) + } + + fun actionNotificationsShow(show: Boolean) { + FooLog.i(TAG, "actionNotificationsShow($show)") + if (show) { + performGlobalAction(GLOBAL_ACTION_NOTIFICATIONS) + } else { + performGlobalAction(GLOBAL_ACTION_DISMISS_NOTIFICATION_SHADE) + } + } + + fun actionTakeScreenshot() { + FooLog.i(TAG, "actionTakeScreenshot()") + performGlobalAction(GLOBAL_ACTION_TAKE_SCREENSHOT) + } + + fun actionKeycodeHeadsetHook() { + FooLog.i(TAG, "actionKeycodeHeadsetHook()") + performGlobalAction(GLOBAL_ACTION_KEYCODE_HEADSETHOOK) + } + + @RequiresApi(36) + fun actionMediaPlayPause() { + FooLog.i(TAG, "actionMediaPlayPause()") + performGlobalAction(GLOBAL_ACTION_MEDIA_PLAY_PAUSE) + } + + // ------------------------------------------------------------------------- + // Lifecycle + // ------------------------------------------------------------------------- + + override fun onCreate() { + FooLog.v(TAG, "#ACCESSIBILITY onCreate()") + super.onCreate() + } + + override fun onDestroy() { + FooLog.v(TAG, "#ACCESSIBILITY onDestroy()") + super.onDestroy() + instance = null + } + + override fun onServiceConnected() { + FooLog.v(TAG, "#ACCESSIBILITY onServiceConnected()") + super.onServiceConnected() + instance = this + } + + override fun onInterrupt() { + FooLog.v(TAG, "#ACCESSIBILITY onInterrupt()") + } + + // ------------------------------------------------------------------------- + // Snapshot state + delegates + // ------------------------------------------------------------------------- + + private var lastSnapshot: List = emptyList() + + private val searchQueue = ShadeRowSearchQueue( + delays = delays, + getWindows = { windows }, + getLastSnapshot = { lastSnapshot }, + openShade = { actionNotificationsShow(true) }, + closeShade = { actionNotificationsShow(false) }, + ) + + private val debugScan = DebugShadeScan( + delays = delays, + getWindows = { windows }, + getLastSnapshot = { lastSnapshot }, + openShade = { actionNotificationsShow(true) }, + closeShade = { actionNotificationsShow(false) }, + ) + + fun findRowForAppLabel( + appLabel: String, + shadeAlreadyOpen: Boolean, + callback: (List?) -> Unit, + ) = searchQueue.findRowForAppLabel(appLabel, shadeAlreadyOpen, callback) + + fun debugFullShadeScan() = debugScan.debugFullShadeScan() + + // ------------------------------------------------------------------------- + // Accessibility events + // ------------------------------------------------------------------------- + + override fun onAccessibilityEvent(event: AccessibilityEvent) { + val packageName = event.packageName?.toString() ?: return + + // + // For privacy/security reasons we only process `com.android.systemui` events + // + if (packageName != NotificationShadeSnapshot.COM_ANDROID_SYSTEMUI) return + + // + // For privacy/security reasons we only process notification relevant events + // + if (event.eventType !in ACCESSIBILITY_EVENTS_NOTIFICATION_RELEVANT) return + + val eventTypeName = AccessibilityEvent.eventTypeToString(event.eventType) + val root = event.source?.window?.root ?: run { + if (VERBOSE_LOG_ROOT_NULL) { + FooLog.v(TAG, "#ACCESSIBILITY onAccessibilityEvent[$eventTypeName]: source window root is null") + } + return + } + + val snapshot = NotificationShadeSnapshot.snapshotShade(eventTypeName, root) + // Debounce (ignore) identical snapshot collections + if (snapshot == lastSnapshot) return + lastSnapshot = snapshot + + if (snapshot.isEmpty()) { + if (VERBOSE_LOG_SHADE_EMPTY) { + FooLog.v(TAG, "#ACCESSIBILITY onAccessibilityEvent[$eventTypeName]: shade snapshot (empty)") + } + } else { + val sb = StringBuilder() + sb.append("#ACCESSIBILITY onAccessibilityEvent[$eventTypeName]: shade snapshot (${snapshot.size} rows):") + snapshot.forEachIndexed { i, row -> sb.append("\n [$i] $row") } + FooLog.v(TAG, sb.toString()) + } + + searchQueue.advancePendingRowSearch() + } +} diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/MyNotificationListenerService.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/MyNotificationListenerService.kt index 28d15b5..4a7fa6e 100644 --- a/mobile/src/main/java/llc/lookatwhataicando/notifai/MyNotificationListenerService.kt +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/MyNotificationListenerService.kt @@ -12,10 +12,17 @@ import com.smartfoo.android.core.FooListenerAutoStartManager import com.smartfoo.android.core.FooString import com.smartfoo.android.core.logging.FooLog import com.smartfoo.android.core.notification.FooNotification +import com.smartfoo.android.core.platform.FooPlatformUtils +import com.smartfoo.android.core.texttospeech.FooTextToSpeechBuilder import llc.lookatwhataicando.notifai.notification.NotificationParserUtils +import llc.lookatwhataicando.notifai.notification.ObscuredNotification +import llc.lookatwhataicando.notifai.notification.ObscuredNotificationLogger +import llc.lookatwhataicando.notifai.notification.ResolutionOutcome import llc.lookatwhataicando.notifai.notification.parsers.NotificationParser import llc.lookatwhataicando.notifai.notification.parsers.NotificationParserAndroidMessages import llc.lookatwhataicando.notifai.notification.parsers.NotificationParserDiscord +import llc.lookatwhataicando.notifai.startup.Requirement +import java.util.concurrent.ConcurrentHashMap class MyNotificationListenerService : NotificationListenerService() { companion object { @@ -24,6 +31,9 @@ class MyNotificationListenerService : NotificationListenerService() { @Suppress("SimplifyBooleanWithConstants", "KotlinConstantConditions", "RedundantSuppression", "UNREACHABLE_CODE") private val LOG_NOTIFICATION = true && BuildConfig.DEBUG + /** Window for sibling (child) notifications to arrive and cancel the accessibility lookup. */ + private const val PENDING_LOOKUP_DELAY_MS = 300L + fun isNotificationListenerEnabled(context: Context) = FooNotification.isNotificationListenerEnabled(context, MyNotificationListenerService::class) @@ -39,6 +49,7 @@ class MyNotificationListenerService : NotificationListenerService() { fun requestNotificationListenerUnbind(context: Context) { FooNotification.requestNotificationListenerUnbind(context, MyNotificationListenerService::class) } + } interface NotificationParserServiceCallbacks { fun onNotificationParsed(parser: NotificationParser) @@ -64,6 +75,19 @@ class MyNotificationListenerService : NotificationListenerService() { private val listenerManager = FooListenerAutoStartManager(this) + /** + * Returns true only when every hard [Requirement] — including POST_NOTIFICATIONS, + * NOTIFICATION_LISTENER, and ACCESSIBILITY_SERVICE — is satisfied. + * + * The system binds this service as soon as NOTIFICATION_LISTENER is granted, regardless of + * other requirements. Both [onListenerConnected] and [onNotificationPosted] check this before + * doing any work, so the listener stays completely idle — no live notifications are spoken and + * no launch catch-up occurs — until Accessibility (and all other requirements) are also + * granted. This is intentional: the accessibility fallback path is required for correct + * startup behaviour, so there is no partial mode to offer. + */ + private fun areRequirementsMet() = Requirement.missing(this).isEmpty() + private val notificationParsers = mutableMapOf() private fun addNotificationParser(notificationParser: NotificationParser) { @@ -118,6 +142,8 @@ class MyNotificationListenerService : NotificationListenerService() { private val activeNotificationsSnapshot = ActiveNotificationsSnapshot(this) + private val handler = Handler(Looper.getMainLooper()) + fun initializeActiveNotifications() { if (LOG_NOTIFICATION) { FooLog.v(TAG, "#NOTIFICATION +initializeActiveNotifications()") @@ -141,6 +167,15 @@ class MyNotificationListenerService : NotificationListenerService() { FooLog.d(TAG, "#NOTIFICATION onListenerConnected()") } super.onListenerConnected() + if (!areRequirementsMet()) { + FooLog.d(TAG, "#NOTIFICATION onListenerConnected: requirements not met; unbinding until ready") + // Unbind so the system stops delivering notifications. + // OperationalScreen calls requestNotificationListenerRebind() once all requirements are met, + // at which point onListenerConnected fires again and initializeActiveNotifications() + // catches up via getActiveNotifications() — no notifications are permanently lost. + requestNotificationListenerUnbind(this) + return + } /* * Delay initialization to avoid system_server process binder transaction issues during the onListenerConnected callback: * ``` @@ -192,6 +227,11 @@ class MyNotificationListenerService : NotificationListenerService() { return } + if (!areRequirementsMet()) { + FooLog.v(TAG, "onNotificationPosted: requirements not met; ignoring") + return + } + val packageName = NotificationParserUtils.getPackageName(sbn) //FooLog.v(TAG, "onNotificationPosted: packageName=${FooString.quote(packageName)}") @@ -205,7 +245,112 @@ class MyNotificationListenerService : NotificationListenerService() { when (result) { NotificationParser.NotificationParseResult.Unparsable -> FooLog.w(TAG, "onNotificationPosted: Unparsable StatusBarNotification") - else -> {} + NotificationParser.NotificationParseResult.ParsedEmpty -> { + // Content was not readable via NLS — schedule a deferred accessibility lookup + // in case the content is visible in the notification shade. + schedulePendingLookup(sbn) + } + else -> { + // Content arrived via NLS — cancel any pending accessibility lookup for this package. + cancelPendingLookup(packageName) + } + } + } + + /** Pending accessibility lookups keyed by packageName. Runnables fire after PENDING_LOOKUP_DELAY_MS. */ + private val pendingLookups = ConcurrentHashMap() + + private fun schedulePendingLookup(sbn: StatusBarNotification) { + val packageName = sbn.packageName + // Cancel any existing pending lookup for this package (e.g. rapid repost). + cancelPendingLookup(packageName) + + val appLabel = FooPlatformUtils.getApplicationName(this, packageName) ?: packageName + FooLog.d(TAG, "#ACCESSIBILITY schedulePendingLookup: packageName=$packageName appLabel=$appLabel delay=${PENDING_LOOKUP_DELAY_MS}ms") + + val runnable = Runnable { + pendingLookups.remove(packageName) + val accessibility = MyAccessibilityService.instance + if (accessibility == null) { + FooLog.w(TAG, "#ACCESSIBILITY schedulePendingLookup: accessibility unavailable for $packageName") + ObscuredNotificationLogger.log( + ObscuredNotification( + packageName = packageName, + appLabel = appLabel, + notificationFlags = sbn.notification.flags, + postedAtMs = sbn.postTime, + resolutionOutcome = ResolutionOutcome.ACCESSIBILITY_UNAVAILABLE, + ) + ) + return@Runnable + } + if (MyAccessibilityService.DEBUG_FULL_SCAN_MODE) { + accessibility.debugFullShadeScan() + return@Runnable + } + accessibility.findRowForAppLabel( + appLabel = appLabel, + shadeAlreadyOpen = false, + callback = { rows -> + val outcome = if (!rows.isNullOrEmpty()) ResolutionOutcome.FOUND else ResolutionOutcome.NOT_FOUND + ObscuredNotificationLogger.log( + ObscuredNotification( + packageName = packageName, + appLabel = appLabel, + notificationFlags = sbn.notification.flags, + postedAtMs = sbn.postTime, + resolutionOutcome = outcome, + ) + ) + if (!rows.isNullOrEmpty()) { + speakShadeRows(appLabel, rows) + } else { + FooLog.i(TAG, "#ACCESSIBILITY schedulePendingLookup: no shade rows found for $appLabel") + } + } + ) + } + + pendingLookups[packageName] = runnable + handler.postDelayed(runnable, PENDING_LOOKUP_DELAY_MS) + // Stale-obscured (confirmed, e.g. com.google.android.apps.dynamite / Google Chat): + // Live — GROUP_SUMMARY is immediately followed by a content-bearing child + // (CHAT_CHIME, MessagingStyle, etc.) within milliseconds. That child arrives + // at onNotificationPosted, hits the `else` branch above, and cancels this + // runnable before it fires. MyAccessibilityService is NOT involved. + // Stale — initializeActiveNotifications iterates getActiveNotifications(). The + // content-bearing child has already been dismissed; only the empty + // GROUP_SUMMARY is active. No cancel arrives → this runnable fires → + // MyAccessibilityService opens the shade and reads the row. + // This is the primary reason the Accessibility permission is required. + // + // Always-obscured (theoretical, unconfirmed): + // Some apps may never post a content-bearing sibling — not even for live delivery. + // This runnable would always fire for those packages. Theory: if this class of app + // exists, it is limited to system-signed / AOSP / Google apps. See + // ObscuredNotificationLogger and notification/parsers/README.md for detection. + } + + private fun cancelPendingLookup(packageName: String) { + val runnable = pendingLookups.remove(packageName) ?: return + handler.removeCallbacks(runnable) + FooLog.d(TAG, "#ACCESSIBILITY cancelPendingLookup: cancelled pending accessibility lookup for $packageName") + } + + private fun speakShadeRows(appLabel: String, rows: List) { + val builder = FooTextToSpeechBuilder(appLabel) + for (row in rows) { + row.title?.let { builder.appendSilenceWordBreak(); builder.appendSpeech(it) } + if (row.messages.isNotEmpty()) { + row.messages.forEach { builder.appendSilenceWordBreak(); builder.appendSpeech(it) } + } else { + row.text?.let { builder.appendSilenceWordBreak(); builder.appendSpeech(it) } + } + } + if (builder.numberOfParts > 1) { + parserCallbacks.textToSpeech.speak(builder) + } else { + FooLog.w(TAG, "speakShadeRows: shade rows for $appLabel have no speakable content: $rows") } } diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/NotificationShadeSnapshot.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/NotificationShadeSnapshot.kt new file mode 100644 index 0000000..a226c7f --- /dev/null +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/NotificationShadeSnapshot.kt @@ -0,0 +1,313 @@ +package llc.lookatwhataicando.notifai + +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import com.smartfoo.android.core.logging.FooLog +import llc.lookatwhataicando.notifai.NotificationShadeSnapshot.DEBUG_DUMP_WINDOWS +import llc.lookatwhataicando.notifai.NotificationShadeSnapshot.NOTIFICATION_CONTAINER_IDS +import llc.lookatwhataicando.notifai.NotificationShadeSnapshot.NOTIFICATION_ROW_ID_SUFFIXES +import llc.lookatwhataicando.notifai.NotificationShadeSnapshot.extractRowData + +object NotificationShadeSnapshot { + private val TAG = FooLog.TAG(NotificationShadeSnapshot::class) + +// --------------------------------------------------------------------------- +// Shared constants — used by this file, ShadeRowSearchQueue, and DebugShadeScan +// --------------------------------------------------------------------------- + + internal const val COM_ANDROID_SYSTEMUI = "com.android.systemui" + + /** + * Known notification shade container view IDs, ordered by preference. First match wins. + * When the heuristic fallback fires instead, it logs the actual ID found — paste that in here + * to avoid the heuristic cost on future runs. + * + * To add a new OEM: run with DEBUG_DUMP_WINDOWS = true, find the scrollable container that + * holds the notification rows, and add its full viewIdResourceName here. + */ + internal val NOTIFICATION_CONTAINER_IDS = listOf( + "com.android.systemui:id/notification_stack_scroller", // AOSP / Pixel (confirmed) + "com.android.systemui:id/notification_panel", // unconfirmed OEM variant + "com.android.systemui:id/notification_list", // unconfirmed OEM variant + ) + + /** + * Known notification row view ID suffixes, ordered by preference. + * Suffix matching tolerates differing package prefixes across OEMs. + * When the heuristic fallback fires, it logs the actual suffix found — add it here. + * + * To add a new OEM: run with DEBUG_DUMP_WINDOWS = true, find the direct children of the + * container that represent individual notifications, and add their viewIdResourceName suffix. + */ + internal val NOTIFICATION_ROW_ID_SUFFIXES = listOf( + "expandableNotificationRow", // AOSP / Pixel (confirmed) + ) + + private const val DEBUG_DUMP_WINDOWS = false + private const val VERBOSE_LOG_CONTAINER_NOT_FOUND = false + +// --------------------------------------------------------------------------- +// ShadeRow +// --------------------------------------------------------------------------- + + /** + * Structured snapshot of a single notification row in the notification shade. + * + * Fields are extracted by view ID suffix — suffix matching tolerates OEM/android:id/ variations. + * appName — from .../app_name_text (e.g. "Discord", "Chat") + * title — from .../title or .../notification_title (channel, contact, or subject) + * sender — from .../message_name (individual sender in a group/messaging notification) + * messages — all .../message_text values in order (one per bubble line) + * text — from .../notification_text or .../big_text or .../text when messages is empty + * time — from .../time (Android provides the long form "6 hours ago" vs the UI "6h") + */ + data class ShadeRow( + val appName: String?, + val title: String?, + val sender: String?, + val messages: List, + val text: String?, + val time: String?, + ) { + val isEmpty: Boolean + get() = appName == null && title == null && sender == null && messages.isEmpty() && text == null + } + +// --------------------------------------------------------------------------- +// Snapshot +// --------------------------------------------------------------------------- + + /** + * Single DFS from [root] that simultaneously dumps (when [DEBUG_DUMP_WINDOWS] is true), + * locates the notification shade container, and collects row nodes. + * + * Collecting rows during this traversal — rather than in a separate pass — is essential. + * Each call to [AccessibilityNodeInfo.getChild] can return a new pooled wrapper; a second + * call for the same child index may yield a stale wrapper with an incomplete childCount. + * Row nodes obtained here (first access) have correct childCount, so [extractRowData] can + * reach all siblings including app_name_text and time without needing refresh(). + * + * Container search: Tier 1 = known IDs in [NOTIFICATION_CONTAINER_IDS]. + * Tier 2 = structural heuristic (scrollable, majority-clickable children). + * Row collection: Tier 1 = direct children matching [NOTIFICATION_ROW_ID_SUFFIXES]. + * Tier 2 = all clickable direct children of the container. + */ + internal fun snapshotShade(eventTypeName: String, root: AccessibilityNodeInfo): List { + if (DEBUG_DUMP_WINDOWS) { + FooLog.v( + TAG, + "snapshotShade[$eventTypeName]: pkg=${root.packageName} id=${root.viewIdResourceName}" + ) + } + + val rowNodes = mutableListOf() + + fun logNode(node: AccessibilityNodeInfo, depth: Int) { + val indent = " ".repeat(depth) + val id = node.viewIdResourceName?.removePrefix("$COM_ANDROID_SYSTEMUI:id/") ?: "" + val cls = node.className?.toString()?.substringAfterLast('.') ?: "" + val text = node.text?.toString()?.replace('\n', '↵') ?: "" + val desc = node.contentDescription?.toString()?.replace('\n', '↵') ?: "" + val flags = buildString { + if (node.isClickable) append("click ") + if (node.isScrollable) append("scroll ") + if (node.isCheckable) append("check ") + if (node.isEnabled) append("enabled ") + if (node.isVisibleToUser) append("visible ") + }.trimEnd() + FooLog.v(TAG, buildString { + append("$indent[$cls] id=$id") + if (text.isNotEmpty()) append(" text=\"$text\"") + if (desc.isNotEmpty()) append(" desc=\"$desc\"") + if (flags.isNotEmpty()) append(" [$flags]") + }) + } + + // Returns true when the container is found and rows are collected, so the caller can + // short-circuit when DEBUG_DUMP_WINDOWS is false. + fun traverse(node: AccessibilityNodeInfo, depth: Int): Boolean { + if (DEBUG_DUMP_WINDOWS) logNode(node, depth) + + if (node.viewIdResourceName in NOTIFICATION_CONTAINER_IDS) { + for (i in 0 until node.childCount) { + node.getChild(i)?.let { child -> + if (NOTIFICATION_ROW_ID_SUFFIXES.any { child.viewIdResourceName?.endsWith(it) == true }) { + rowNodes.add(child) + } + if (DEBUG_DUMP_WINDOWS) traverse(child, depth + 1) + } + } + return true + } + + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + if (traverse(child, depth + 1) && !DEBUG_DUMP_WINDOWS) return true + } + return false + } + + traverse(root, 0) + + if (rowNodes.isEmpty()) { + val heuristic = findNotificationContainerHeuristic(root) + if (heuristic != null) { + FooLog.w( + TAG, + "snapshotShade: heuristic container id=${heuristic.viewIdResourceName} — add to NOTIFICATION_CONTAINER_IDS" + ) + val matched = (0 until heuristic.childCount) + .mapNotNull { heuristic.getChild(it) } + .filter { it.isClickable } + if (matched.isNotEmpty()) { + val sampleSuffix = + (matched.first().viewIdResourceName ?: "").substringAfterLast('/') + FooLog.w( + TAG, + "snapshotShade: heuristic rows sample suffix=$sampleSuffix — add to NOTIFICATION_ROW_ID_SUFFIXES" + ) + rowNodes.addAll(matched) + } + } else if (VERBOSE_LOG_CONTAINER_NOT_FOUND) { + FooLog.w(TAG, "snapshotShade: no notification container found") + } + } + + return rowNodes.mapNotNull { extractRowData(it).takeUnless { row -> row.isEmpty } } + } + + private fun findNotificationContainerHeuristic(node: AccessibilityNodeInfo): AccessibilityNodeInfo? { + if (node.isScrollable) { + val childCount = node.childCount + if (childCount > 0) { + val clickable = + (0 until childCount).count { node.getChild(it)?.isClickable == true } + if (clickable * 2 >= childCount) return node + } + } + for (i in 0 until node.childCount) { + node.getChild(i)?.let { findNotificationContainerHeuristic(it) }?.let { return it } + } + return null + } + + /** + * Walks a notification row's subtree and extracts structured fields by view ID suffix. + * Suffix matching tolerates both `android:id/` and `com.android.systemui:id/` prefixes. + */ + private fun extractRowData(row: AccessibilityNodeInfo): ShadeRow { + var appName: String? = null + var title: String? = null + var sender: String? = null + val messages = mutableListOf() + var text: String? = null + var time: String? = null + + fun walk(node: AccessibilityNodeInfo) { + val id = node.viewIdResourceName + val nodeText = node.text?.toString() + if (!nodeText.isNullOrBlank()) { + when { + id?.endsWith("/app_name_text") == true -> appName = nodeText + id?.endsWith("/title") == true -> if (title == null) title = nodeText + id?.endsWith("/notification_title") == true -> if (title == null) title = + nodeText + + id?.endsWith("/message_name") == true -> sender = nodeText + id?.endsWith("/message_text") == true -> messages.add(nodeText) + id?.endsWith("/big_text") == true -> if (text == null) text = nodeText + id?.endsWith("/notification_text") == true -> if (text == null) text = nodeText + id?.endsWith("/text") == true -> if (text == null) text = nodeText + id?.endsWith("/time") == true -> if (time == null) time = nodeText + } + } + for (i in 0 until node.childCount) { + node.getChild(i)?.let { walk(it) } + } + } + + walk(row) + + return ShadeRow( + appName = appName, + title = title, + sender = sender, + messages = messages, + text = text, + time = time, + ) + } + +// --------------------------------------------------------------------------- +// Tree traversal utilities — used by ShadeRowSearchQueue and DebugShadeScan +// --------------------------------------------------------------------------- + + internal fun getLiveRowNodes(windows: List?): List { + windows ?: return emptyList() + for (window in windows) { + val root = window.root ?: continue + if (root.packageName?.toString() != COM_ANDROID_SYSTEMUI) continue + val rows = mutableListOf() + if (collectRowNodes(root, rows) && rows.isNotEmpty()) return rows + } + return emptyList() + } + + internal fun collectRowNodes( + node: AccessibilityNodeInfo, + rows: MutableList + ): Boolean { + if (node.viewIdResourceName in NOTIFICATION_CONTAINER_IDS) { + for (i in 0 until node.childCount) { + node.getChild(i)?.let { child -> + if (NOTIFICATION_ROW_ID_SUFFIXES.any { child.viewIdResourceName?.endsWith(it) == true }) { + rows.add(child) + } + } + } + return true + } + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + if (collectRowNodes(child, rows)) return true + } + return false + } + + internal fun findContainerNode(root: AccessibilityNodeInfo): AccessibilityNodeInfo? { + if (root.viewIdResourceName in NOTIFICATION_CONTAINER_IDS) return root + for (i in 0 until root.childCount) { + val child = root.getChild(i) ?: continue + findContainerNode(child)?.let { return it } + } + return null + } + + internal fun getLiveContainerNode(windows: List?): AccessibilityNodeInfo? { + windows ?: return null + for (window in windows) { + val root = window.root ?: continue + if (root.packageName?.toString() != COM_ANDROID_SYSTEMUI) continue + findContainerNode(root)?.let { return it } + } + return null + } + + internal fun findRawRowWithAppName( + appLabel: String, + windows: List?, + ): AccessibilityNodeInfo? = + getLiveRowNodes(windows).firstOrNull { hasAppNameInSubtree(it, appLabel) } + + internal fun hasAppNameInSubtree(node: AccessibilityNodeInfo, appLabel: String): Boolean { + if (node.viewIdResourceName?.endsWith("/app_name_text") == true && + node.text?.toString()?.equals(appLabel, ignoreCase = true) == true + ) return true + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + if (hasAppNameInSubtree(child, appLabel)) return true + } + return false + } + +} \ No newline at end of file diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/ShadeDelays.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/ShadeDelays.kt new file mode 100644 index 0000000..2c7e1c7 --- /dev/null +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/ShadeDelays.kt @@ -0,0 +1,24 @@ +package llc.lookatwhataicando.notifai + +import llc.lookatwhataicando.notifai.ShadeDelays.Companion.FAST +import llc.lookatwhataicando.notifai.ShadeDelays.Companion.SLOW + + +/** + * Timing constants for the notification shade accessibility state machine. + * + * Use [SLOW] while debugging (visual observation of each step). + * Use [FAST] in production once minimum working delays are confirmed. + * Toggle via [MyAccessibilityService.DEBUG_SLOW_MODE]. + */ +data class ShadeDelays( + val shadeSettle: Long, + val preExpand: Long, + val scrollSettle: Long, + val preClose: Long, +) { + companion object { + val FAST = ShadeDelays(shadeSettle = 200L, preExpand = 50L, scrollSettle = 50L, preClose = 50L) + val SLOW = ShadeDelays(shadeSettle = 2000L, preExpand = 1000L, scrollSettle = 1500L, preClose = 4000L) + } +} diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/ShadeRowSearchQueue.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/ShadeRowSearchQueue.kt new file mode 100644 index 0000000..1189fe5 --- /dev/null +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/ShadeRowSearchQueue.kt @@ -0,0 +1,271 @@ +package llc.lookatwhataicando.notifai + +import android.os.Handler +import android.os.Looper +import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo +import com.smartfoo.android.core.logging.FooLog + +/** + * Serializes asynchronous notification shade row searches for [MyAccessibilityService]. + * + * Each [findRowForAppLabel] call is either started immediately (if no search is active) or + * queued behind the current one. When a search finishes, the open shade is handed directly to + * the next queued search — no close/reopen cycle. + * + * This prevents multiple near-simultaneous calls — e.g. Chat and Google both firing at startup + * within 30 ms of each other — from silently cancelling each other. + */ +internal class ShadeRowSearchQueue( + private val delays: ShadeDelays, + private val getWindows: () -> List?, + private val getLastSnapshot: () -> List, + private val openShade: () -> Unit, + private val closeShade: () -> Unit, +) { + companion object { + private val TAG = FooLog.TAG(ShadeRowSearchQueue::class) + private const val MAX_SCROLL_ATTEMPTS = 10 + } + + private val mainHandler = Handler(Looper.getMainLooper()) + + /** + * State for a single in-flight search for a notification row by app label. + * + * @param appLabel Human-readable label to match against [NotificationShadeSnapshot.ShadeRow.appName]. + * @param callback Invoked on the main thread with all found rows (or null if none). + * @param shadeOpenedByUs True if this search is responsible for closing the shade when done. + * Transferred to the next queued search so the last one closes it. + * @param readyToScan False while waiting for the shade-open settle delay; true once the + * settle runnable fires. + * @param scrollAttemptsLeft Remaining [AccessibilityNodeInfo.ACTION_SCROLL_FORWARD] calls + * allowed before giving up on off-screen rows. + * @param rowExpanded True after we performed one expand on the target row, preventing + * re-expanding on the next snapshot check. + * @param settling True while a [mainHandler.postDelayed] settle window is in-flight + * (pre-expand pause, post-expand settle, or post-scroll settle). + * Blocks [advancePendingRowSearch] from reacting during that window. + */ + private data class PendingRowSearch( + val appLabel: String, + val callback: (List?) -> Unit, + var shadeOpenedByUs: Boolean, + var readyToScan: Boolean, + var scrollAttemptsLeft: Int = MAX_SCROLL_ATTEMPTS, + var rowExpanded: Boolean = false, + var settling: Boolean = false, + ) + + /** Head is the active search; all others wait for the active one to finish. */ + private val queue: ArrayDeque = ArrayDeque() + + /** + * Initiates a search for the shade row whose [NotificationShadeSnapshot.ShadeRow.appName] matches [appLabel]. + * + * If another search is active, this one is queued and will start automatically when the + * current one finishes ([finishSearch] hands the open shade to the next entry). + * + * @param appLabel Label to search for (e.g. "Chat"). + * @param shadeAlreadyOpen True if the shade is already visible (skip open + settle delay). + * @param callback Called on the main thread with the matched rows or null. + */ + fun findRowForAppLabel( + appLabel: String, + shadeAlreadyOpen: Boolean, + callback: (List?) -> Unit, + ) { + FooLog.i(TAG, "#ACCESSIBILITY findRowForAppLabel: appLabel=$appLabel shadeAlreadyOpen=$shadeAlreadyOpen queueSize=${queue.size}") + + val shadeIsOpen = shadeAlreadyOpen || queue.isNotEmpty() + val search = PendingRowSearch( + appLabel = appLabel, + callback = callback, + shadeOpenedByUs = !shadeIsOpen, + readyToScan = shadeIsOpen, + ) + + if (queue.isNotEmpty()) { + FooLog.i(TAG, "#ACCESSIBILITY findRowForAppLabel: queuing $appLabel behind active search for ${queue.first().appLabel}") + queue.addLast(search) + return + } + + queue.addLast(search) + + if (!shadeAlreadyOpen) { + openShade() + mainHandler.postDelayed({ + val s = queue.firstOrNull() ?: return@postDelayed + if (s !== search) return@postDelayed + s.readyToScan = true + selfAdvance(s) + }, delays.shadeSettle) + } else { + selfAdvance(search) + } + } + + /** + * Called from [MyAccessibilityService.onAccessibilityEvent] after every snapshot update. + * Delegates to [selfAdvance] so that event-driven and timer-driven paths share one code path. + */ + fun advancePendingRowSearch() { + val search = queue.firstOrNull() ?: return + if (search.settling) return + selfAdvance(search) + } + + /** + * Central state-machine advance. Called from event-driven [advancePendingRowSearch] AND + * proactively from every settle-timer callback so the search never stalls waiting for a + * passive accessibility event. + * + * Priority order: + * 1. If target app is in the snapshot and the row is collapsed, expand it (with optional + * pre-expand pause), then re-enter to finish. + * 2. If target app is in the snapshot and expanded (or no expand needed), finish the search. + * 3. If no match yet, expand the next collapsed visible row to reveal more content. + * 4. When all visible rows are expanded, scroll and repeat. + * 5. When scroll is exhausted or fails, give up. + */ + private fun selfAdvance(search: PendingRowSearch) { + if (!search.readyToScan) return + + val match = getLastSnapshot().firstOrNull { it.appName.equals(search.appLabel, ignoreCase = true) } + if (match != null) { + if (!search.rowExpanded) { + // The row may be a collapsed group summary. Expand it first so we capture all + // child notifications. ACTION_EXPAND is locale-agnostic. We require EXPAND present + // AND COLLAPSE absent to avoid acting on transitional or ambiguous actionList state. + val rawNode = NotificationShadeSnapshot.findRawRowWithAppName(search.appLabel, getWindows()) + val isCollapsed = rawNode?.actionList?.let { a -> + a.any { it.id == AccessibilityNodeInfo.ACTION_EXPAND } && + a.none { it.id == AccessibilityNodeInfo.ACTION_COLLAPSE } + } == true + if (isCollapsed) { + if (delays.preExpand > 0) { + FooLog.v(TAG, "#ACCESSIBILITY selfAdvance: pausing ${delays.preExpand}ms before expanding ${search.appLabel}") + search.settling = true + mainHandler.postDelayed({ + val s = queue.firstOrNull() ?: return@postDelayed + if (s !== search) return@postDelayed + s.settling = false + val freshNode = NotificationShadeSnapshot.findRawRowWithAppName(s.appLabel, getWindows()) + val freshIsCollapsed = freshNode?.actionList?.let { a -> + a.any { it.id == AccessibilityNodeInfo.ACTION_EXPAND } && + a.none { it.id == AccessibilityNodeInfo.ACTION_COLLAPSE } + } == true + if (freshIsCollapsed) { + FooLog.v(TAG, "#ACCESSIBILITY selfAdvance: now expanding ${s.appLabel}") + freshNode!!.performAction(AccessibilityNodeInfo.ACTION_EXPAND) + s.rowExpanded = true + } + selfAdvance(s) + }, delays.preExpand) + } else { + FooLog.v(TAG, "#ACCESSIBILITY selfAdvance: expanding ${search.appLabel} before reading content") + rawNode!!.performAction(AccessibilityNodeInfo.ACTION_EXPAND) + search.rowExpanded = true + // Fast mode: snapshot may not yet reflect the expand. Let the next + // accessibility event drive the re-check via advancePendingRowSearch. + } + return + } + } + val allMatches = getLastSnapshot().filter { it.appName.equals(search.appLabel, ignoreCase = true) } + FooLog.i(TAG, "#ACCESSIBILITY selfAdvance: found ${search.appLabel} (${allMatches.size} rows)") + finishSearch(allMatches) + return + } + + if (!tryExpandNextRow(search)) { + val container = NotificationShadeSnapshot.getLiveContainerNode(getWindows()) + if (search.scrollAttemptsLeft > 0 && container != null && + container.performAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD)) { + search.scrollAttemptsLeft-- + FooLog.v(TAG, "#ACCESSIBILITY selfAdvance: scrolled (${search.scrollAttemptsLeft} attempts left)") + if (delays.scrollSettle > 0) { + search.settling = true + mainHandler.postDelayed({ + val s = queue.firstOrNull() ?: return@postDelayed + if (s !== search) return@postDelayed + s.settling = false + selfAdvance(s) + }, delays.scrollSettle) + } else { + selfAdvance(search) + } + } else { + FooLog.i(TAG, "#ACCESSIBILITY selfAdvance: scroll exhausted or container null — giving up on ${search.appLabel}") + finishSearch() + } + } + } + + /** + * Re-fetches live row nodes and performs ACTION_EXPAND on the first collapsed row found. + * Locale-agnostic — detected via actionList (EXPAND present, COLLAPSE absent). Nodes are + * fetched fresh on every call; no stale references are held between async delays. + * + * Returns true if an expand was performed, false if all visible rows are already expanded + * (caller should scroll or give up). After any settle delay, proactively calls [selfAdvance] + * so the search advances without waiting for a passive accessibility event. + */ + private fun tryExpandNextRow(search: PendingRowSearch): Boolean { + for (row in NotificationShadeSnapshot.getLiveRowNodes(getWindows())) { + val actions = row.actionList + if (actions.any { it.id == AccessibilityNodeInfo.ACTION_EXPAND } && + actions.none { it.id == AccessibilityNodeInfo.ACTION_COLLAPSE }) { + val expanded = row.performAction(AccessibilityNodeInfo.ACTION_EXPAND) + FooLog.v(TAG, "#ACCESSIBILITY tryExpandNextRow: ACTION_EXPAND result=$expanded for ${row.viewIdResourceName}") + if (delays.preExpand > 0) { + search.settling = true + mainHandler.postDelayed({ + val s = queue.firstOrNull() ?: return@postDelayed + if (s !== search) return@postDelayed + s.settling = false + selfAdvance(s) + }, delays.preExpand) + } + return true + } + } + return false + } + + private fun finishSearch(results: List = emptyList()) { + val search = queue.removeFirstOrNull() ?: return + val nextSearch = queue.firstOrNull() + FooLog.i(TAG, "#ACCESSIBILITY finishSearch: appLabel=${search.appLabel} rows=${results.size} queueRemaining=${queue.size}") + + if (nextSearch != null) { + // Hand the open shade to the next search immediately (no close/reopen cycle). + // Transfer shade-close responsibility so the last search in the chain closes it. + if (search.shadeOpenedByUs) nextSearch.shadeOpenedByUs = true + nextSearch.readyToScan = true + // Defer selfAdvance to the next looper iteration so the accessibility tree has time + // to reflect the just-completed search's final expand (avoids stale ACTION_EXPAND + // on an already-expanded row being seen by the incoming search's first scan). + nextSearch.settling = true + mainHandler.post { + val s = queue.firstOrNull() ?: return@post + if (s !== nextSearch) return@post + s.settling = false + selfAdvance(s) + } + search.callback(results.takeIf { it.isNotEmpty() }) + } else { + val closeAndCallback = { + if (search.shadeOpenedByUs) closeShade() + search.callback(results.takeIf { it.isNotEmpty() }) + } + if (delays.preClose > 0) { + FooLog.v(TAG, "#ACCESSIBILITY finishSearch: holding shade open ${delays.preClose}ms for observation (DEBUG_SLOW_MODE)") + mainHandler.postDelayed(closeAndCallback, delays.preClose) + } else { + closeAndCallback() + } + } + } +} diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/ObscuredNotification.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/ObscuredNotification.kt new file mode 100644 index 0000000..f4fc7f8 --- /dev/null +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/ObscuredNotification.kt @@ -0,0 +1,43 @@ +package llc.lookatwhataicando.notifai.notification + +/** + * Describes a notification whose content is obscured from [android.service.notification.NotificationListenerService] + * — the NLS payload contains no title, text, or extras — but whose content IS visible in the + * notification shade and readable via [MyAccessibilityService]. + * + * ## Sub-classes + * + * ### Stale-obscured (confirmed, common) + * During live delivery the app posts a GROUP_SUMMARY immediately followed by a content-bearing + * child notification (e.g. `CHAT_CHIME`, `MessagingStyle`) within milliseconds. NLS reads the + * child normally; the 300 ms cancellation window in `schedulePendingLookup` prevents the + * accessibility path from firing. The GROUP_SUMMARY becomes obscured only during stale + * catch-up (`initializeActiveNotifications`) — the child has already been dismissed, only the + * empty GROUP_SUMMARY remains. + * + * Confirmed: `com.google.android.apps.dynamite` (Google Chat). + * + * ### Always-obscured (theoretical, unconfirmed) + * Some apps may never post a content-bearing sibling — not even for live delivery. Content is + * only ever available in the shade. The accessibility path would fire for both live and stale + * notifications. If this category exists, it is theorized to be limited to system-signed / + * AOSP / Google apps. See `notification/parsers/README.md` for detection guidance. + * + * Currently uncharacterized: `com.google.android.googlequicksearchbox` (Google). + */ +data class ObscuredNotification( + val packageName: String, + val appLabel: String, + val notificationFlags: Int, + val postedAtMs: Long, + val resolutionOutcome: ResolutionOutcome, +) + +enum class ResolutionOutcome { + /** Accessibility found and read the shade row for this notification. */ + FOUND, + /** Accessibility was available but no matching row was found in the shade. */ + NOT_FOUND, + /** MyAccessibilityService.instance was null — permission not granted. */ + ACCESSIBILITY_UNAVAILABLE, +} diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/ObscuredNotificationLogger.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/ObscuredNotificationLogger.kt new file mode 100644 index 0000000..23a30f9 --- /dev/null +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/ObscuredNotificationLogger.kt @@ -0,0 +1,31 @@ +package llc.lookatwhataicando.notifai.notification + +import com.smartfoo.android.core.logging.FooLog +import llc.lookatwhataicando.notifai.notification.ObscuredNotificationLogger.sink + +/** + * Sink for [ObscuredNotification] telemetry. + * + * Register a custom sink via [ObscuredNotificationLogger.sink] to route events to a backend + * (Firebase Firestore, Analytics, a local DB, etc.). + * The default sink writes to logcat only. + */ +fun interface ObscuredNotificationSink { + fun log(event: ObscuredNotification) +} + +/** + * Central logger for [ObscuredNotification] events. + * + * The [sink] is swappable at runtime — swap in a real backend implementation (e.g. Firebase) + * at app startup via [ObscuredNotificationLogger.sink] = myFirebaseSink. + */ +object ObscuredNotificationLogger { + private val TAG = FooLog.TAG(ObscuredNotificationLogger::class) + + var sink: ObscuredNotificationSink = ObscuredNotificationSink { event -> + FooLog.i(TAG, "#OBSCURED log: pkg=${event.packageName} app=${event.appLabel} outcome=${event.resolutionOutcome} flags=0x${event.notificationFlags.toString(16)}") + } + + fun log(event: ObscuredNotification) = sink.log(event) +} diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/parsers/NotificationParser.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/parsers/NotificationParser.kt index 6b0f36e..12452cb 100644 --- a/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/parsers/NotificationParser.kt +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/parsers/NotificationParser.kt @@ -3,8 +3,6 @@ package llc.lookatwhataicando.notifai.notification.parsers import android.app.Notification import android.content.Context import android.service.notification.StatusBarNotification -import android.view.View -import android.widget.TextView import androidx.core.app.NotificationCompat import com.smartfoo.android.core.FooString import com.smartfoo.android.core.logging.FooLog @@ -135,10 +133,6 @@ abstract class NotificationParser(private val hashtag: String, protected val par val category = notification.category FooLog.v(TAG, "defaultOnNotificationPosted: category=$category") - if (textToSpeechManager == null) { - return NotificationParseResult.ParsedIgnored - } - if (builder.numberOfParts == 1) { // // Found nothing in the above [currently commented out] bespoke parsing, @@ -184,11 +178,18 @@ abstract class NotificationParser(private val hashtag: String, protected val par } } + // TODO: Clean up this ugly logic... + + if (textToSpeechManager == null) { + FooLog.w(TAG, "defaultOnNotificationPosted: textToSpeechManager == null; ignore and return ParsedIgnored") + return NotificationParseResult.ParsedIgnored + } + if (builder.numberOfParts > 1) { textToSpeechManager.speak(builder) } else { - FooLog.w(TAG, "defaultOnNotificationPosted: No notification parts found; ignoring") - return NotificationParseResult.ParsedIgnored + FooLog.w(TAG, "defaultOnNotificationPosted: No notification parts found; ignore and return ParsedEmpty") + return NotificationParseResult.ParsedEmpty } return NotificationParseResult.DefaultWithTickerText @@ -203,6 +204,7 @@ abstract class NotificationParser(private val hashtag: String, protected val par DefaultWithTickerText, DefaultWithoutTickerText, Unparsable, + ParsedEmpty, ParsedIgnored, ParsedHandled, } diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/parsers/README.md b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/parsers/README.md new file mode 100644 index 0000000..12c48e4 --- /dev/null +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/notification/parsers/README.md @@ -0,0 +1,122 @@ +# Notification Parsing — Ground Truth + +## Two delivery paths + +### Path 1 — NotificationListenerService (NLS), `MyNotificationListenerService` + +The primary path. The system delivers every `StatusBarNotification` to `onNotificationPosted`. +`defaultOnNotificationPosted` extracts title, text, ticker, and `MessagingStyle` messages from +the notification extras and speaks them via TTS. + +This covers the vast majority of apps and all well-behaved notifications. + +### Path 2 — AccessibilityService, `MyAccessibilityService` + +A fallback for notifications whose content is **not present in the NLS payload** but **is +visible in the notification shade**. The accessibility tree exposes the rendered shade rows, +including content that the app only populates in the visual layout. + +--- + +## "Obscured" notifications (`ObscuredNotification`) + +Some apps post a `GROUP_SUMMARY` notification whose NLS payload contains no title, text, or +extras — only metadata. The actual message content is only accessible by reading the expanded +notification row in the shade via the accessibility tree. + +There are two meaningfully different sub-classes of obscured notification: + +### Stale-obscured (confirmed, common) + +For live delivery the GROUP_SUMMARY is **immediately followed** by a content-bearing child +notification (`CHAT_CHIME`, `MessagingStyle`, etc.) within milliseconds. NLS sees both and +reads the child normally — no accessibility involvement. + +The GROUP_SUMMARY becomes obscured only when it is **stale**: the app has restarted, +`initializeActiveNotifications` iterates `getActiveNotifications()`, and the content-bearing +child has already been dismissed. Only the empty GROUP_SUMMARY remains. The 300 ms +`PENDING_LOOKUP_DELAY_MS` cancellation window elapses with no sibling, so the accessibility +path fires. + +**Confirmed stale-obscured packages:** +- `com.google.android.apps.dynamite` (Google Chat) — live GROUP_SUMMARY is immediately + followed by a `CHAT_CHIME` child; only the stale GROUP_SUMMARY reaches accessibility. + +**Live flow (no accessibility involvement):** +1. GROUP_SUMMARY arrives → `ParsedIgnored` → `schedulePendingLookup(300 ms)` +2. Content-bearing child arrives → `cancelPendingLookup` fires before 300 ms elapses +3. Child spoken via normal NLS path. + +**Stale/catch-up flow (accessibility required):** +1. GROUP_SUMMARY → `ParsedIgnored` → `schedulePendingLookup(300 ms)` +2. 300 ms elapses with no cancel → `MyAccessibilityService.findRowForAppLabel()` +3. Shade opens → rows scanned (expanding collapsed rows, scrolling for off-screen) → + content read from accessibility tree → spoken via TTS. + +**This is the primary scenario requiring the Accessibility permission.** + +### Always-obscured (theoretical, unconfirmed) + +The hypothesis: some apps **never** post a content-bearing sibling — not even for live +delivery. The GROUP_SUMMARY is the only notification they ever post for a given message. +Content is only ever visible in the shade. These packages would require the accessibility +path for **both** live and stale delivery; the `PENDING_LOOKUP_DELAY_MS` cancellation window +would never fire regardless of app state. + +**Theory:** if this class of app exists, it is almost certainly limited to **system-signed / +AOSP / Google apps** — third-party apps generally carry readable content in their NLS payload +or in a sibling notification. System apps have additional delivery channels (e.g. FCM data +messages delivered directly to the app process) and may use the notification solely as a +visual shade badge. + +**Currently uncharacterized** (appeared in production logs, live vs. stale behavior not yet +confirmed): +- `com.google.android.googlequicksearchbox` (Google) + +**Detection signal:** `ObscuredNotificationLogger` will consistently log `FOUND` outcomes for +a given `packageName` even when notifications arrive in real-time (not only at app startup). +A package that triggers the accessibility path on live notifications — not just catch-up — is +a candidate for the always-obscured category. + +--- + +## Open questions / assumptions to validate + +### Does any known app always require accessibility, even for live delivery? + +For **Google Chat** (`com.google.android.apps.dynamite`): confirmed stale-obscured only. +Live delivery always includes a content-bearing sibling. The 300 ms cancellation window +reliably prevents the accessibility path from firing for live notifications. + +**Unconfirmed:** whether any package consistently reaches accessibility even for freshly +arriving live notifications. The leading theory is that if such a class exists, it is limited +to system-signed / AOSP / Google apps that have alternative delivery channels and use the +notification only for the visual badge. + +**Action needed:** monitor `ObscuredNotificationLogger` output for any `packageName` that +consistently fires the accessibility lookup with `FOUND` outcomes during live notification +delivery (i.e. not only at app startup). Such a package is a candidate for always-obscured +classification and may need a bespoke `NotificationParser` or a dedicated accessibility search +strategy. + +--- + +## Adding a new parser + +1. Create `NotificationParserXxx.kt` extending `NotificationParser`. +2. Override `packageName` and `onNotificationPosted`. +3. Register in `MyNotificationListenerService.addNotificationParsers()`. + +If the app only posts `GROUP_SUMMARY` with no content extras, the accessibility path handles +it automatically — no custom parser required unless bespoke field extraction is needed. + +--- + +## Constants + +| Constant | Location | Value | Purpose | +|---|---|---|---| +| `PENDING_LOOKUP_DELAY_MS` | `MyNotificationListenerService` | 300 ms | Window for content-bearing child to cancel the accessibility lookup | +| `ShadeDelays.FAST.shadeSettle` | `ShadeDelays` | 600 ms | Shade animation settle time before scanning (production) | +| `ShadeDelays.SLOW.shadeSettle` | `ShadeDelays` | 3000 ms | Shade animation settle time before scanning (debug/observation) | +| `MAX_SCROLL_ATTEMPTS` | `ShadeRowSearchQueue`, `DebugShadeScan` | 10 | Max `ACTION_SCROLL_FORWARD` calls before giving up on off-screen rows | diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/NotificationListenerEnabledMonitor.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/NotificationListenerEnabledMonitor.kt index 8e77ec8..de3ad58 100644 --- a/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/NotificationListenerEnabledMonitor.kt +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/NotificationListenerEnabledMonitor.kt @@ -6,11 +6,11 @@ import android.os.Handler import android.os.Looper import android.provider.Settings import androidx.core.app.NotificationManagerCompat +import com.smartfoo.android.core.notification.FooNotification import kotlinx.coroutines.channels.awaitClose import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.distinctUntilChanged -import com.smartfoo.android.core.notification.FooNotification /** * Reactive Flow that emits whenever the system's enabled_notification_listeners diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/StartupCoordinator.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/StartupCoordinator.kt index 56a2245..7174efb 100644 --- a/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/StartupCoordinator.kt +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/startup/StartupCoordinator.kt @@ -1,8 +1,12 @@ package llc.lookatwhataicando.notifai.startup import android.app.Application +import android.content.Context import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope +import com.smartfoo.android.core.notification.FooNotification +import com.smartfoo.android.core.permission.FooPermission +import com.smartfoo.android.core.platform.FooPlatformUtils import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow @@ -11,16 +15,57 @@ import kotlinx.coroutines.flow.drop import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch +import llc.lookatwhataicando.notifai.MyAccessibilityService import llc.lookatwhataicando.notifai.MyNotificationListenerService -import com.smartfoo.android.core.notification.FooNotification -import com.smartfoo.android.core.permission.FooPermission /** * Requirement = hard gate. App cannot function without these. */ enum class Requirement { POST_NOTIFICATIONS, // Runtime permission (API 33+ only) - NOTIFICATION_LISTENER // Settings-mediated special access + NOTIFICATION_LISTENER, // Settings-mediated special access + + /** + * Settings-mediated special access. + * + * ## Why this is required + * + * **Live notifications** (arriving while the app is running) are handled entirely by + * [MyNotificationListenerService] (NLS). When an app like Google Chat posts a silent + * GROUP_SUMMARY followed by a content-bearing child notification, NLS sees both within + * milliseconds and the 300 ms cancellation window in [MyNotificationListenerService] + * ensures the content is spoken via the normal NLS path. No accessibility involvement. + * + * **Launch / catch-up** is where [MyAccessibilityService] becomes necessary. When the app + * starts and iterates [android.service.notification.NotificationListenerService.getActiveNotifications], + * the content-bearing child notifications may already be gone — only the empty GROUP_SUMMARY + * remains active. NLS cannot read its content. [MyAccessibilityService] opens the notification + * shade, expands collapsed rows, and reads the content via the accessibility tree. + * + * ## Secondary capability + * + * [MyAccessibilityService] also provides global navigation actions (open/dismiss shade, + * back, home, screenshot) that NLS does not have access to. + * + * ## Practical impact + * + * This is a **hard requirement**. [MyNotificationListenerService.areRequirementsMet] gates all + * NLS processing on every [Requirement] being satisfied, including this one. Without + * Accessibility the NLS stays idle: no live notifications are spoken and no launch catch-up + * occurs. The UI reflects this by blocking the operational screen until this is granted. + */ + ACCESSIBILITY_SERVICE; + + companion object { + fun missing(context: Context): Set = buildSet { + if (!FooNotification.isPostNotificationsPermissionGranted(context)) + add(POST_NOTIFICATIONS) + if (!MyNotificationListenerService.isNotificationListenerEnabled(context)) + add(NOTIFICATION_LISTENER) + if (!FooPlatformUtils.isAccessibilityServiceEnabled(context, MyAccessibilityService::class.java)) + add(ACCESSIBILITY_SERVICE) + } + } } /** @@ -110,10 +155,7 @@ class StartupCoordinator(private val app: Application) : AndroidViewModel(app) { * - After ListenerEnabledMonitor emits (via collect above) */ fun recheck() { - val missing = buildSet { - if (!FooNotification.isPostNotificationsPermissionGranted(app)) add(Requirement.POST_NOTIFICATIONS) - if (!MyNotificationListenerService.isNotificationListenerEnabled(app)) add(Requirement.NOTIFICATION_LISTENER) - } + val missing = Requirement.missing(app) val advisories = buildSet { if (!FooPermission.isIgnoringBatteryOptimizations(app)) add(Advisory.BATTERY_OPTIMIZATION) diff --git a/mobile/src/main/java/llc/lookatwhataicando/notifai/ui/theme/Theme.kt b/mobile/src/main/java/llc/lookatwhataicando/notifai/ui/theme/Theme.kt index a4df47e..d81a366 100644 --- a/mobile/src/main/java/llc/lookatwhataicando/notifai/ui/theme/Theme.kt +++ b/mobile/src/main/java/llc/lookatwhataicando/notifai/ui/theme/Theme.kt @@ -1,6 +1,5 @@ package llc.lookatwhataicando.notifai.ui.theme -import android.os.Build import androidx.compose.foundation.isSystemInDarkTheme import androidx.compose.material3.MaterialTheme import androidx.compose.material3.darkColorScheme diff --git a/mobile/src/main/res/values-night/themes.xml b/mobile/src/main/res/values-night/themes.xml index 5588900..267da05 100644 --- a/mobile/src/main/res/values-night/themes.xml +++ b/mobile/src/main/res/values-night/themes.xml @@ -1,4 +1,4 @@ - +