fix(backend): stop task-manager event queries from flipping to a quadratic generic plan

backend/infrahub/prefect_server/database.py

@@ -12,9 +12,13 @@
 
 
 async def query_events(
-    session: AsyncSession, filter: EventFilter, page_size: int = INTERACTIVE_PAGE_SIZE, offset: int | None = None
+    session: AsyncSession,
+    filter: EventFilter,
+    page_size: int = INTERACTIVE_PAGE_SIZE,
+    offset: int | None = None,
+    include_total: bool = True,
 ) -> tuple[list[ReceivedEvent], int]:
-    count = await raw_count_events(session, filter)  # type: ignore[attr-defined]
+    count = await raw_count_events(session, filter) if include_total else 0  # type: ignore[attr-defined]
     page = await read_events(session, filter, limit=page_size, offset=offset)  # type: ignore[attr-defined]
     events = [ReceivedEvent.model_validate(e, from_attributes=True) for e in page]
     return events, count

backend/infrahub/prefect_server/events.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter
 from fastapi.param_functions import Depends
 from prefect.server.database import PrefectDBInterface, provide_database_interface
+from sqlalchemy import text
 
 from .database import query_events
 from .models import InfrahubEventfilterInput, InfrahubEventPage
@@ -18,8 +19,19 @@ async def read_events(
     event_filter.filter.set_prefix()
 
     async with db.session_context() as session:
+        if session.bind is not None and session.bind.dialect.name == "postgresql":
+            # After five executions of a prepared statement Postgres may switch it to a
+            # generic plan chosen without seeing the parameter values; for these event
+            # filters that plan degrades to a quadratic join that stalls for tens of
+            # seconds. Force per-execution planning for this transaction only, so the
+            # rest of the Prefect server keeps its prepared-statement plan caching.
+            await session.execute(text("SET LOCAL plan_cache_mode = force_custom_plan"))
         events, total = await query_events(
-            session=session, filter=event_filter.filter, page_size=event_filter.limit, offset=event_filter.offset
+            session=session,
+            filter=event_filter.filter,
+            page_size=event_filter.limit,
+            offset=event_filter.offset,
+            include_total=event_filter.include_total,
         )
 
         return InfrahubEventPage(

backend/infrahub/prefect_server/models.py

@@ -26,3 +26,7 @@ class InfrahubEventfilterInput(BaseModel):
     limit: int = Field(default=50)
     filter: InfrahubEventFilter = Field(default_factory=InfrahubEventFilter.default)
     offset: int | None = Field(default=None)
+    include_total: bool = Field(
+        default=True,
+        description="When false, skip the unbounded count query and report total=0; the paged events are unaffected",
+    )

backend/infrahub/task_manager/event/query.py

@@ -396,17 +396,30 @@ async def query_events(
         limit: int,
         filters: InfrahubEventFilter,
         offset: int | None = None,
+        include_total: bool = True,
     ) -> PrefectEventResponse:
-        body = {"limit": limit, "filter": filters.model_dump(mode="json", exclude_none=True), "offset": offset}
+        body = {
+            "limit": limit,
+            "filter": filters.model_dump(mode="json", exclude_none=True),
+            "offset": offset,
+            "include_total": include_total,
+        }
 
         # Retry due to https://github.com/PrefectHQ/prefect/issues/16299
-        for _ in range(1, 5):
+        for attempt in range(1, 5):
             prefect_error: PrefectHTTPStatusError | None = None
             try:
                 response = await client._client.post("/infrahub/events/filter", json=body)
                 break
             except PrefectHTTPStatusError as exc:
                 prefect_error = exc
+                # Each failed attempt can hide up to a full task-manager request timeout,
+                # so a silent loop here turns into a multi-minute stall for the caller.
+                log.warning(
+                    "Event query to the task manager failed, retrying",
+                    attempt=attempt,
+                    status_code=exc.response.status_code,
+                )
                 await asyncio.sleep(0.1)
 
         if prefect_error:
@@ -438,8 +451,14 @@ async def query(
             # returning data that will only be discarded
             limit = 1
 
+        # The count is an unbounded aggregate over the whole filter window and is by far
+        # the most expensive part of the endpoint, so only ask for it when selected.
+        include_total = "count" in fields
+
         async with get_client(sync_client=False) as client:
-            response = await cls.query_events(client=client, filters=event_filter, limit=limit, offset=offset)
+            response = await cls.query_events(
+                client=client, filters=event_filter, limit=limit, offset=offset, include_total=include_total
+            )
             nodes = [{"node": event.to_graphql()} for event in response.events]
 
         return {"count": response.count, "edges": nodes}

changelog/+events-query-generic-plan-stall.fixed.md

@@ -0,0 +1 @@
+Fixed intermittent multi-second to multi-minute stalls of the activity log. The task manager's Postgres connections switched the event queries to a generic prepared-statement plan after five executions, which degraded the unbounded count that backs every activity-log page from milliseconds to a quadratic join. The event-filter endpoint now forces per-execution planning for its own transaction only, and the count is only computed when the GraphQL query selects it.

dev/knowledge/backend/events.md

@@ -124,6 +124,31 @@ Events can be queried through:
 - **REST API**: `/infrahub/events/filter` endpoint
 - **Prefect Client**: Direct Prefect event API access
 
+### Query-path performance constraints
+
+The `/infrahub/events/filter` endpoint runs two SQL statements against the task manager's
+Postgres: an unbounded `count(*)` over the whole filter window and the `LIMIT`-ed page read.
+Two hard-earned constraints apply to this path:
+
+- **The count is only computed when the caller asks for it.** The count aggregates every
+  matching row while the page read stops at the page size, so the count dominates the
+  endpoint's cost. The GraphQL resolver requests it (`include_total`) only when the query
+  selects `count` — the activity-log UI does not, so its page loads skip the aggregate
+  entirely. Keep that property when extending the endpoint.
+- **The endpoint forces per-execution planning** (`SET LOCAL plan_cache_mode =
+  force_custom_plan` at the start of its transaction). After five executions of a
+  prepared statement, Postgres may switch it to a *generic* plan chosen without seeing
+  the parameter values. For these event filters — a wide `occurred` window plus a JSON
+  label match against `event_resources` — the generic plan degrades from a linear hash
+  join to a quadratic nested loop (measured: 5 ms → 3.5 s on a 5k-event table, growing
+  quadratically). The flip is per pool connection and per statement, which made the
+  resulting stalls look like a once-a-week CI flake: one pool connection runs the
+  pathological plan while its siblings answer in milliseconds. `SET LOCAL` scopes the
+  countermeasure to this transaction only — the rest of the Prefect server keeps its
+  prepared-statement plan caching — at the cost of replanning these two queries per
+  request (~1.5 ms). Do not remove it without re-checking the event queries' plans under
+  `plan_cache_mode = force_generic_plan`.
+
 ## Key Locations
 
 | Component | Location |