Fix stats search

.gitignore

@@ -34,5 +34,6 @@ sbt.json
 /yarn.lock
 .metals
 .vscode
+.cursor
 .bloop
 metals.sbt

Changelog.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## v0.44.0
+
+*Unreleased*
+
+### 🚀 Features
+
+- Add `statsProfile` to search stats (`full`, `general`, `fields`). The
+  dashboard uses lighter `general` and `fields` requests instead of
+  repeating the full statistics query.
+
+### 🐛 Bug Fixes
+
+- Improve performance of custom field statistics on large databases by
+  scoping `custom_field_value` lookups to matching items only.
+
+### 💚 Maintenance
+
+- Document search stats profiles in the OpenAPI spec. The `general`
+  profile still computes tag clouds; it omits dimension list aggregates
+  and heavy field statistics.
+
 ## v0.43.0
 
 *Mar 15, 2025*

build.sbt

@@ -241,6 +241,12 @@ val openapiScalaSettings = Seq(
         field =>
           field
             .copy(typeDef = TypeDef("SearchMode", Imports("docspell.common.SearchMode")))
+      case "statsprofile" =>
+        field =>
+          field
+            .copy(typeDef =
+              TypeDef("StatsProfile", Imports("docspell.common.StatsProfile"))
+            )
       case "duration" =>
         field =>
           field

modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala

@@ -69,7 +69,8 @@ trait OSearch[F[_]] {
 
   /** Run multiple database calls with the give query to collect a summary. */
   def searchSummary(
-      today: Option[LocalDate]
+      today: Option[LocalDate],
+      profile: StatsProfile = StatsProfile.Full
   )(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
 
   /** Parses a query string and creates a `Query` object, to be used with the other
@@ -208,7 +209,8 @@ object OSearch {
         } yield resolved
 
       def searchSummary(
-          today: Option[LocalDate]
+          today: Option[LocalDate],
+          profile: StatsProfile
       )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
         fulltextQuery match {
           case Some(ftq) =>
@@ -220,7 +222,7 @@ object OSearch {
               results <- WeakAsync.liftK[F, ConnectionIO].use { nat =>
                 val tempTable = temporaryFtsTable(ftq, nat)
                 store.transact(
-                  tempTable.flatMap(tt => QItem.searchStats(date, tt.some)(q))
+                  tempTable.flatMap(tt => runSearchStats(profile, date, tt.some)(q))
                 )
               }
             } yield results
@@ -229,7 +231,18 @@ object OSearch {
             OptionT
               .fromOption(today)
               .getOrElseF(Timestamp.current[F].map(_.toUtcDate))
-              .flatMap(date => store.transact(QItem.searchStats(date, None)(q)))
+              .flatMap(date => store.transact(runSearchStats(profile, date, None)(q)))
+        }
+
+      private def runSearchStats(
+          profile: StatsProfile,
+          date: LocalDate,
+          ftsTable: Option[RFtsResult.Table]
+      )(q: Query): ConnectionIO[SearchSummary] =
+        profile match {
+          case StatsProfile.Full    => QItem.searchStats(date, ftsTable)(q)
+          case StatsProfile.General => QItem.searchStatsGeneral(date, ftsTable)(q)
+          case StatsProfile.Fields  => QItem.searchStatsFields(date, ftsTable)(q)
         }
 
       private def createFtsQuery(

modules/common/src/main/scala/docspell/common/StatsProfile.scala

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import cats.data.NonEmptyList
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait StatsProfile { self: Product =>
+
+  final def name: String =
+    productPrefix.toLowerCase
+}
+
+object StatsProfile {
+
+  final case object Full extends StatsProfile
+  final case object General extends StatsProfile
+  final case object Fields extends StatsProfile
+
+  val default: StatsProfile = Full
+
+  def fromString(str: String): Either[String, StatsProfile] =
+    str.toLowerCase match {
+      case "full"    => Right(Full)
+      case "general" => Right(General)
+      case "fields"  => Right(Fields)
+      case _         => Left(s"Invalid stats profile: $str")
+    }
+
+  val all: NonEmptyList[StatsProfile] =
+    NonEmptyList.of(Full, General, Fields)
+
+  def unsafe(str: String): StatsProfile =
+    fromString(str).fold(sys.error, identity)
+
+  implicit val jsonDecoder: Decoder[StatsProfile] =
+    Decoder.decodeString.emap(fromString)
+  implicit val jsonEncoder: Encoder[StatsProfile] =
+    Encoder.encodeString.contramap(_.name)
+}

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -3038,8 +3038,45 @@ paths:
       description: |
         Instead of returning the results of a query, uses it to return
         a summary.
+
+        The default stats profile is `full`, which preserves the
+        behavior of existing API clients. Prefer
+        `POST /sec/item/searchStats/{profile}` to select a profile
+        explicitly.
+
+        Profile precedence: path segment, then `statsProfile` in the
+        JSON body, then `full`.
+
+        | Profile | count | tagCloud | fieldStats | folderStats | dimension lists | fieldCount etc. |
+        |---------|-------|----------|------------|-------------|-----------------|-----------------|
+        | full | yes | yes | yes | yes | yes | no |
+        | general | yes | yes | empty | empty | empty | yes |
+        | fields | yes | empty | yes | empty | empty | no |
+
+        Empty arrays in the response do not imply zero items in the
+        database; they indicate that the selected profile does not
+        populate that section.
+
+        The optional `statsProfile` query parameter on this POST route
+        is deprecated; use a path segment instead. The query parameter
+        remains supported on GET.
       security:
         - authTokenHeader: []
+      parameters:
+        - name: statsProfile
+          in: query
+          required: false
+          deprecated: true
+          description: |
+            Deprecated on POST. Use `POST /sec/item/searchStats/{profile}`
+            or the `statsProfile` property in the request body instead.
+          schema:
+            type: string
+            format: statsprofile
+            enum:
+              - full
+              - general
+              - fields
       requestBody:
         content:
           application/json:
@@ -3060,12 +3097,64 @@ paths:
       summary: Get basic statistics about search results.
       description: |
         Instead of returning the results of a query, uses it to return
-        a summary.
+        a summary. Use the optional `statsProfile` query parameter to
+        select `full`, `general`, or `fields`. Defaults to `full`.
       security:
         - authTokenHeader: []
       parameters:
         - $ref: "#/components/parameters/q"
         - $ref: "#/components/parameters/searchMode"
+        - name: statsProfile
+          in: query
+          required: false
+          schema:
+            type: string
+            format: statsprofile
+            enum:
+              - full
+              - general
+              - fields
+      responses:
+        422:
+          description: BadRequest
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SearchStats"
+
+  /sec/item/searchStats/{profile}:
+    post:
+      operationId: "sec-item-search-stats-profile-post"
+      tags: [ Item Search ]
+      summary: Get search statistics for a specific profile.
+      description: |
+        Same as `POST /sec/item/searchStats`, but the stats profile is
+        selected by path segment (`full`, `general`, or `fields`). The
+        path profile takes precedence over `statsProfile` in the JSON
+        body.
+
+        See `POST /sec/item/searchStats` for a description of what each
+        profile populates in the `SearchStats` response.
+      security:
+        - authTokenHeader: []
+      parameters:
+        - name: profile
+          in: path
+          required: true
+          schema:
+            type: string
+            format: statsprofile
+            enum:
+              - full
+              - general
+              - fields
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/ItemQuery"
       responses:
         422:
           description: BadRequest
@@ -7167,6 +7256,19 @@ components:
           type: string
           description: |
             A query searching the contents of documents.
+        statsProfile:
+          type: string
+          format: statsprofile
+          enum:
+            - full
+            - general
+            - fields
+          default: full
+          description: |
+            Controls how much work searchStats performs. The full
+            profile returns all statistics; general returns counts
+            suitable for dashboard summaries; fields returns only
+            custom field statistics.
     MoveAttachment:
       description: |
         Data to move an attachment to another position.
@@ -7568,7 +7670,10 @@ components:
           format: ident
     SearchStats:
       description: |
-        A summary of search results.
+        A summary of search results. All profiles return the same schema
+        shape, but only populate the sections relevant to the selected
+        stats profile. Empty arrays do not imply zero items in the
+        database.
       required:
         - count
         - tagCloud
@@ -7611,6 +7716,31 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/IdRefStats"
+        fieldCount:
+          type: integer
+          format: int32
+          description: |
+            Number of distinct non-numeric custom fields that have at
+            least one value on items matching the query. Only set by
+            the `general` stats profile.
+        orgCount:
+          type: integer
+          format: int32
+          description: |
+            Number of distinct correspondent organizations in results.
+            Populated by the general stats profile.
+        personCount:
+          type: integer
+          format: int32
+          description: |
+            Number of distinct persons (correspondent or concerned) in
+            results. Populated by the general stats profile.
+        equipCount:
+          type: integer
+          format: int32
+          description: |
+            Number of distinct equipment references in results.
+            Populated by the general stats profile.
 
     ItemInsights:
       description: |

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -48,7 +48,11 @@ trait Conversions {
       sum.corrOrgs.map(mkIdRefStats),
       sum.corrPers.map(mkIdRefStats),
       sum.concPers.map(mkIdRefStats),
-      sum.concEquip.map(mkIdRefStats)
+      sum.concEquip.map(mkIdRefStats),
+      sum.fieldCount,
+      sum.orgCount,
+      sum.personCount,
+      sum.equipCount
     )
 
   def mkIdRefStats(s: IdRefCount): IdRefStats =

modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala

@@ -12,7 +12,7 @@ import docspell.backend.ops.OFolder.FolderOrder
 import docspell.backend.ops.OOrganization.{OrganizationOrder, PersonOrder}
 import docspell.backend.ops.OTag.TagOrder
 import docspell.common.ContactKind
-import docspell.common.SearchMode
+import docspell.common.{SearchMode, StatsProfile}
 
 import org.http4s.ParseFailure
 import org.http4s.QueryParamDecoder
@@ -34,6 +34,11 @@ object QueryParam {
       SearchMode.fromString(str).left.map(s => ParseFailure(str, s))
     )
 
+  implicit val statsProfileDecoder: QueryParamDecoder[StatsProfile] =
+    QueryParamDecoder[String].emap(str =>
+      StatsProfile.fromString(str).left.map(s => ParseFailure(str, s))
+    )
+
   implicit val tagOrderDecoder: QueryParamDecoder[TagOrder] =
     QueryParamDecoder[String].emap(str =>
       TagOrder.parse(str).left.map(s => ParseFailure(str, s))
@@ -78,6 +83,8 @@ object QueryParam {
   object Offset extends OptionalQueryParamDecoderMatcher[Int]("offset")
   object WithDetails extends OptionalQueryParamDecoderMatcher[Boolean]("withDetails")
   object SearchKind extends OptionalQueryParamDecoderMatcher[SearchMode]("searchMode")
+  object StatsProfileOpt
+      extends OptionalQueryParamDecoderMatcher[StatsProfile]("statsProfile")
   object TagSort extends OptionalQueryParamDecoderMatcher[TagOrder]("sort")
   object EquipSort extends OptionalQueryParamDecoderMatcher[EquipmentOrder]("sort")
   object OrgSort extends OptionalQueryParamDecoderMatcher[OrganizationOrder]("sort")