diff --git a/.gitignore b/.gitignore index 29bc856f42..a68f154c52 100644 --- a/.gitignore +++ b/.gitignore @@ -34,5 +34,6 @@ sbt.json /yarn.lock .metals .vscode +.cursor .bloop metals.sbt \ No newline at end of file diff --git a/Changelog.md b/Changelog.md index f5d8685075..7a313083ea 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,26 @@ # Changelog +## v0.44.0 + +*Unreleased* + +### 🚀 Features + +- Add `statsProfile` to search stats (`full`, `general`, `fields`). The + dashboard uses lighter `general` and `fields` requests instead of + repeating the full statistics query. + +### 🐛 Bug Fixes + +- Improve performance of custom field statistics on large databases by + scoping `custom_field_value` lookups to matching items only. + +### 💚 Maintenance + +- Document search stats profiles in the OpenAPI spec. The `general` + profile still computes tag clouds; it omits dimension list aggregates + and heavy field statistics. + ## v0.43.0 *Mar 15, 2025* diff --git a/build.sbt b/build.sbt index bd6db9e43b..a417c1942b 100644 --- a/build.sbt +++ b/build.sbt @@ -241,6 +241,12 @@ val openapiScalaSettings = Seq( field => field .copy(typeDef = TypeDef("SearchMode", Imports("docspell.common.SearchMode"))) + case "statsprofile" => + field => + field + .copy(typeDef = + TypeDef("StatsProfile", Imports("docspell.common.StatsProfile")) + ) case "duration" => field => field diff --git a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala index 29c600b05d..be4ddc5734 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala @@ -69,7 +69,8 @@ trait OSearch[F[_]] { /** Run multiple database calls with the give query to collect a summary. */ def searchSummary( - today: Option[LocalDate] + today: Option[LocalDate], + profile: StatsProfile = StatsProfile.Full )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] /** Parses a query string and creates a `Query` object, to be used with the other @@ -208,7 +209,8 @@ object OSearch { } yield resolved def searchSummary( - today: Option[LocalDate] + today: Option[LocalDate], + profile: StatsProfile )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] = fulltextQuery match { case Some(ftq) => @@ -220,7 +222,7 @@ object OSearch { results <- WeakAsync.liftK[F, ConnectionIO].use { nat => val tempTable = temporaryFtsTable(ftq, nat) store.transact( - tempTable.flatMap(tt => QItem.searchStats(date, tt.some)(q)) + tempTable.flatMap(tt => runSearchStats(profile, date, tt.some)(q)) ) } } yield results @@ -229,7 +231,18 @@ object OSearch { OptionT .fromOption(today) .getOrElseF(Timestamp.current[F].map(_.toUtcDate)) - .flatMap(date => store.transact(QItem.searchStats(date, None)(q))) + .flatMap(date => store.transact(runSearchStats(profile, date, None)(q))) + } + + private def runSearchStats( + profile: StatsProfile, + date: LocalDate, + ftsTable: Option[RFtsResult.Table] + )(q: Query): ConnectionIO[SearchSummary] = + profile match { + case StatsProfile.Full => QItem.searchStats(date, ftsTable)(q) + case StatsProfile.General => QItem.searchStatsGeneral(date, ftsTable)(q) + case StatsProfile.Fields => QItem.searchStatsFields(date, ftsTable)(q) } private def createFtsQuery( diff --git a/modules/common/src/main/scala/docspell/common/StatsProfile.scala b/modules/common/src/main/scala/docspell/common/StatsProfile.scala new file mode 100644 index 0000000000..ca09883e62 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/StatsProfile.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.data.NonEmptyList + +import io.circe.{Decoder, Encoder} + +sealed trait StatsProfile { self: Product => + + final def name: String = + productPrefix.toLowerCase +} + +object StatsProfile { + + final case object Full extends StatsProfile + final case object General extends StatsProfile + final case object Fields extends StatsProfile + + val default: StatsProfile = Full + + def fromString(str: String): Either[String, StatsProfile] = + str.toLowerCase match { + case "full" => Right(Full) + case "general" => Right(General) + case "fields" => Right(Fields) + case _ => Left(s"Invalid stats profile: $str") + } + + val all: NonEmptyList[StatsProfile] = + NonEmptyList.of(Full, General, Fields) + + def unsafe(str: String): StatsProfile = + fromString(str).fold(sys.error, identity) + + implicit val jsonDecoder: Decoder[StatsProfile] = + Decoder.decodeString.emap(fromString) + implicit val jsonEncoder: Encoder[StatsProfile] = + Encoder.encodeString.contramap(_.name) +} diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index b149fae707..da2e7b4c61 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -3038,8 +3038,45 @@ paths: description: | Instead of returning the results of a query, uses it to return a summary. + + The default stats profile is `full`, which preserves the + behavior of existing API clients. Prefer + `POST /sec/item/searchStats/{profile}` to select a profile + explicitly. + + Profile precedence: path segment, then `statsProfile` in the + JSON body, then `full`. + + | Profile | count | tagCloud | fieldStats | folderStats | dimension lists | fieldCount etc. | + |---------|-------|----------|------------|-------------|-----------------|-----------------| + | full | yes | yes | yes | yes | yes | no | + | general | yes | yes | empty | empty | empty | yes | + | fields | yes | empty | yes | empty | empty | no | + + Empty arrays in the response do not imply zero items in the + database; they indicate that the selected profile does not + populate that section. + + The optional `statsProfile` query parameter on this POST route + is deprecated; use a path segment instead. The query parameter + remains supported on GET. security: - authTokenHeader: [] + parameters: + - name: statsProfile + in: query + required: false + deprecated: true + description: | + Deprecated on POST. Use `POST /sec/item/searchStats/{profile}` + or the `statsProfile` property in the request body instead. + schema: + type: string + format: statsprofile + enum: + - full + - general + - fields requestBody: content: application/json: @@ -3060,12 +3097,64 @@ paths: summary: Get basic statistics about search results. description: | Instead of returning the results of a query, uses it to return - a summary. + a summary. Use the optional `statsProfile` query parameter to + select `full`, `general`, or `fields`. Defaults to `full`. security: - authTokenHeader: [] parameters: - $ref: "#/components/parameters/q" - $ref: "#/components/parameters/searchMode" + - name: statsProfile + in: query + required: false + schema: + type: string + format: statsprofile + enum: + - full + - general + - fields + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/SearchStats" + + /sec/item/searchStats/{profile}: + post: + operationId: "sec-item-search-stats-profile-post" + tags: [ Item Search ] + summary: Get search statistics for a specific profile. + description: | + Same as `POST /sec/item/searchStats`, but the stats profile is + selected by path segment (`full`, `general`, or `fields`). The + path profile takes precedence over `statsProfile` in the JSON + body. + + See `POST /sec/item/searchStats` for a description of what each + profile populates in the `SearchStats` response. + security: + - authTokenHeader: [] + parameters: + - name: profile + in: path + required: true + schema: + type: string + format: statsprofile + enum: + - full + - general + - fields + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ItemQuery" responses: 422: description: BadRequest @@ -7167,6 +7256,19 @@ components: type: string description: | A query searching the contents of documents. + statsProfile: + type: string + format: statsprofile + enum: + - full + - general + - fields + default: full + description: | + Controls how much work searchStats performs. The full + profile returns all statistics; general returns counts + suitable for dashboard summaries; fields returns only + custom field statistics. MoveAttachment: description: | Data to move an attachment to another position. @@ -7568,7 +7670,10 @@ components: format: ident SearchStats: description: | - A summary of search results. + A summary of search results. All profiles return the same schema + shape, but only populate the sections relevant to the selected + stats profile. Empty arrays do not imply zero items in the + database. required: - count - tagCloud @@ -7611,6 +7716,31 @@ components: type: array items: $ref: "#/components/schemas/IdRefStats" + fieldCount: + type: integer + format: int32 + description: | + Number of distinct non-numeric custom fields that have at + least one value on items matching the query. Only set by + the `general` stats profile. + orgCount: + type: integer + format: int32 + description: | + Number of distinct correspondent organizations in results. + Populated by the general stats profile. + personCount: + type: integer + format: int32 + description: | + Number of distinct persons (correspondent or concerned) in + results. Populated by the general stats profile. + equipCount: + type: integer + format: int32 + description: | + Number of distinct equipment references in results. + Populated by the general stats profile. ItemInsights: description: | diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 78babc04bb..dd6dbd2607 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -48,7 +48,11 @@ trait Conversions { sum.corrOrgs.map(mkIdRefStats), sum.corrPers.map(mkIdRefStats), sum.concPers.map(mkIdRefStats), - sum.concEquip.map(mkIdRefStats) + sum.concEquip.map(mkIdRefStats), + sum.fieldCount, + sum.orgCount, + sum.personCount, + sum.equipCount ) def mkIdRefStats(s: IdRefCount): IdRefStats = diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala index 041814cf95..a91f7a4c33 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/QueryParam.scala @@ -12,7 +12,7 @@ import docspell.backend.ops.OFolder.FolderOrder import docspell.backend.ops.OOrganization.{OrganizationOrder, PersonOrder} import docspell.backend.ops.OTag.TagOrder import docspell.common.ContactKind -import docspell.common.SearchMode +import docspell.common.{SearchMode, StatsProfile} import org.http4s.ParseFailure import org.http4s.QueryParamDecoder @@ -34,6 +34,11 @@ object QueryParam { SearchMode.fromString(str).left.map(s => ParseFailure(str, s)) ) + implicit val statsProfileDecoder: QueryParamDecoder[StatsProfile] = + QueryParamDecoder[String].emap(str => + StatsProfile.fromString(str).left.map(s => ParseFailure(str, s)) + ) + implicit val tagOrderDecoder: QueryParamDecoder[TagOrder] = QueryParamDecoder[String].emap(str => TagOrder.parse(str).left.map(s => ParseFailure(str, s)) @@ -78,6 +83,8 @@ object QueryParam { object Offset extends OptionalQueryParamDecoderMatcher[Int]("offset") object WithDetails extends OptionalQueryParamDecoderMatcher[Boolean]("withDetails") object SearchKind extends OptionalQueryParamDecoderMatcher[SearchMode]("searchMode") + object StatsProfileOpt + extends OptionalQueryParamDecoderMatcher[StatsProfile]("statsProfile") object TagSort extends OptionalQueryParamDecoderMatcher[TagOrder]("sort") object EquipSort extends OptionalQueryParamDecoderMatcher[EquipmentOrder]("sort") object OrgSort extends OptionalQueryParamDecoderMatcher[OrganizationOrder]("sort") diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala index 4aea2b8ed9..b450c96e8a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala @@ -26,7 +26,7 @@ import docspell.store.queries.{ListItemWithTags, SearchSummary} import org.http4s.circe.CirceEntityCodec._ import org.http4s.dsl.Http4sDsl -import org.http4s.{HttpRoutes, Response} +import org.http4s.{HttpRoutes, Request, Response} final class ItemSearchPart[F[_]: Async]( searchOps: OSearch[F], @@ -45,7 +45,7 @@ final class ItemSearchPart[F[_]: Async]( QP.Offset(offset) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) => val userQuery = - ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse("")) + ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse(""), None) for { today <- Timestamp.current[F].map(_.toUtcDate) resp <- search(userQuery, today) @@ -62,34 +62,58 @@ final class ItemSearchPart[F[_]: Async]( } yield resp case GET -> Root / `searchStatsPath` :? QP.Query(q) :? - QP.SearchKind(searchMode) => - val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse("")) + QP.SearchKind(searchMode) :? QP.StatsProfileOpt(qpProfile) => + val userQuery = + ItemQuery(None, None, None, searchMode, q.getOrElse(""), qpProfile) for { today <- Timestamp.current[F].map(_.toUtcDate) - resp <- searchStats(userQuery, today) + resp <- searchStats(userQuery, today, qpProfile) } yield resp - case req @ POST -> Root / `searchStatsPath` => - for { - timed <- Duration.stopTime[F] - userQuery <- req.as[ItemQuery] - today <- Timestamp.current[F].map(_.toUtcDate) - resp <- searchStats(userQuery, today) - dur <- timed - _ <- logger.debug(s"Search stats request: ${dur.formatExact}") - } yield resp + case req @ POST -> Root / `searchStatsPath` / "general" => + postSearchStats(req, StatsProfile.General.some) + + case req @ POST -> Root / `searchStatsPath` / "fields" => + postSearchStats(req, StatsProfile.Fields.some) + + case req @ POST -> Root / `searchStatsPath` / "full" => + postSearchStats(req, StatsProfile.Full.some) + + case req @ POST -> Root / `searchStatsPath` :? QP.StatsProfileOpt(qpProfile) => + postSearchStats(req, qpProfile) } - def searchStats(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = { + private def postSearchStats( + req: Request[F], + pathProfile: Option[StatsProfile] + ): F[Response[F]] = + for { + timed <- Duration.stopTime[F] + userQuery <- req.as[ItemQuery] + today <- Timestamp.current[F].map(_.toUtcDate) + resp <- searchStats(userQuery, today, pathProfile) + dur <- timed + _ <- logger.debug(s"Search stats request: ${dur.formatExact}") + } yield resp + + def searchStats( + userQuery: ItemQuery, + today: LocalDate, + qpProfile: Option[StatsProfile] = None + ): F[Response[F]] = { val mode = userQuery.searchMode.getOrElse(SearchMode.Normal) parsedQuery(userQuery, mode) .fold( identity, - res => + res => { + // Profile precedence: path segment, then body statsProfile, then default full. + val profile = + qpProfile.orElse(userQuery.statsProfile).getOrElse(StatsProfile.default) for { - summary <- searchOps.searchSummary(today.some)(res.q, res.ftq) + summary <- searchOps.searchSummary(today.some, profile)(res.q, res.ftq) resp <- Ok(Conversions.mkSearchStats(changeSummary(summary))) } yield resp + } ) } diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index eec2d9164b..a5a6497761 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -298,8 +298,16 @@ object QItem extends FtsSupport { today: LocalDate, coll: CollectiveId, q: ItemQuery.Expr + ): Condition = + queryCondFromExprFor(i, today, coll, q) + + private[queries] def queryCondFromExprFor( + item: RItem.Table, + today: LocalDate, + coll: CollectiveId, + q: ItemQuery.Expr ): Condition = { - val tables = Tables(i, org, pers0, pers1, equip, f, a, m) + val tables = Tables(item, org, pers0, pers1, equip, f, a, m) ItemQueryGenerator.fromExpr(today, tables, coll)(q) } @@ -307,10 +315,18 @@ object QItem extends FtsSupport { today: LocalDate, coll: CollectiveId, cond: Query.QueryCond + ): Condition = + queryConditionFor(i, today, coll, cond) + + private[queries] def queryConditionFor( + item: RItem.Table, + today: LocalDate, + coll: CollectiveId, + cond: Query.QueryCond ): Condition = cond match { case Query.QueryExpr(Some(expr)) => - queryCondFromExpr(today, coll, expr) + queryCondFromExprFor(item, today, coll, expr) case Query.QueryExpr(None) => Condition.unit } @@ -322,7 +338,7 @@ object QItem extends FtsSupport { count <- searchCountSummary(today, ftsTable)(q) tags <- searchTagSummary(today, ftsTable)(q) cats <- searchTagCategorySummary(today, ftsTable)(q) - fields <- searchFieldSummary(today, ftsTable)(q) + fields <- QItemFieldStats.searchFieldSummary(today, ftsTable)(q) folders <- searchFolderSummary(today, ftsTable)(q) orgs <- searchCorrOrgSummary(today, ftsTable)(q) corrPers <- searchCorrPersonSummary(today, ftsTable)(q) @@ -340,6 +356,150 @@ object QItem extends FtsSupport { concEquip ) + def searchStatsGeneral(today: LocalDate, ftsTable: Option[RFtsResult.Table])( + q: Query + ): ConnectionIO[SearchSummary] = + for { + folderIds <- QFolder.getMemberFolders( + q.fix.account.collectiveId, + q.fix.account.userId + ) + folderIdsOpt = Some(folderIds) + count <- searchCountSummary(today, ftsTable, folderIdsOpt)(q) + tags <- searchTagSummary(today, ftsTable)(q) + fieldCount <- QItemFieldStats.searchFieldDefinitionCountForQuery( + q.fix, + today, + q.cond, + ftsTable, + folderIds, + count + ) + orgCount <- searchDistinctOrgCount(today, ftsTable, folderIdsOpt)(q) + personCount <- searchDistinctPersonCount(today, ftsTable, folderIdsOpt)(q) + equipCount <- searchDistinctEquipCount(today, ftsTable, folderIdsOpt)(q) + } yield SearchSummary( + count = count, + tags = tags, + cats = Nil, + fields = Nil, + folders = Nil, + corrOrgs = Nil, + corrPers = Nil, + concPers = Nil, + concEquip = Nil, + fieldCount = fieldCount.some, + orgCount = orgCount.some, + personCount = personCount.some, + equipCount = equipCount.some + ) + + def searchStatsFields(today: LocalDate, ftsTable: Option[RFtsResult.Table])( + q: Query + ): ConnectionIO[SearchSummary] = + QItemFieldStats.resolveStatsItemContext(q.fix, today, q.cond, ftsTable).flatMap { + ctx => + val folderIds = Some(ctx.folderIds) + val coll = q.fix.account.collectiveId + for { + count <- searchCountSummary(today, ftsTable, folderIds)(q) + fields <- QItemFieldStats.searchFieldSummaryFromContext(coll, ctx) + } yield SearchSummary( + count = count, + tags = Nil, + cats = Nil, + fields = fields, + folders = Nil, + corrOrgs = Nil, + corrPers = Nil, + concPers = Nil, + concEquip = Nil + ) + } + + private[queries] def statsItemMatchingIdsSelect( + fix: Query.Fix, + today: LocalDate, + cond: Query.QueryCond, + ftsTable: Option[RFtsResult.Table], + folderIds: Option[Set[Ident]] + ): Select = + statsItemSelect(fix, today, cond, folderIds) + .joinFtsIdOnly(i, ftsTable) + .withSelect(Nel.of(i.id.s)) + + private[queries] def statsItemMatchCount( + fix: Query.Fix, + today: LocalDate, + cond: Query.QueryCond, + ftsTable: Option[RFtsResult.Table], + folderIds: Option[Set[Ident]] + ): ConnectionIO[Int] = + statsItemSelect(fix, today, cond, folderIds) + .joinFtsIdOnly(i, ftsTable) + .withSelect(Nel.of(count(i.id).as("num"))) + .build + .query[Int] + .unique + + private def statsItemFolderConditionFor( + item: RItem.Table, + folderIds: Set[Ident] + ): Condition = + Nel.fromList(folderIds.toList) match { + case None => + item.folder.isNull + case Some(nel) => + or(item.folder.isNull, item.folder.in(nel)) + } + + private def statsItemWhere( + fix: Query.Fix, + today: LocalDate, + coll: CollectiveId, + cond: Query.QueryCond, + folderIds: Option[Set[Ident]] + ): Condition = + statsItemWhereFor(i, fix, today, coll, cond, folderIds) + + private[queries] def statsItemWhereFor( + item: RItem.Table, + fix: Query.Fix, + today: LocalDate, + coll: CollectiveId, + cond: Query.QueryCond, + folderIds: Option[Set[Ident]] + ): Condition = { + val folderCond = folderIds match { + case Some(ids) => + statsItemFolderConditionFor(item, ids) + case None => + or( + item.folder.isNull, + item.folder.in( + QFolder.findMemberFolderIds(fix.account.collectiveId, fix.account.userId) + ) + ) + } + item.cid === coll &&? fix.query.map(qs => + queryCondFromExprFor(item, today, coll, qs) + ) && + folderCond && + queryConditionFor(item, today, coll, cond) + } + + private def statsItemSelect( + fix: Query.Fix, + today: LocalDate, + cond: Query.QueryCond, + folderIds: Option[Set[Ident]] + ): Select.SimpleSelect = + Select( + select(i.id), + from(i), + statsItemWhere(fix, today, fix.account.collectiveId, cond, folderIds) + ) + def searchTagCategorySummary( today: LocalDate, ftsTable: Option[RFtsResult.Table] @@ -394,17 +554,67 @@ object QItem extends FtsSupport { } yield existing ++ other.map(TagCount(_, 0)) } - def searchCountSummary(today: LocalDate, ftsTable: Option[RFtsResult.Table])( - q: Query - ): ConnectionIO[Int] = - findItemsBase(q.fix, today, 0, None).unwrap + def searchCountSummary( + today: LocalDate, + ftsTable: Option[RFtsResult.Table], + folderIds: Option[Set[Ident]] = None + )(q: Query): ConnectionIO[Int] = + statsItemSelect(q.fix, today, q.cond, folderIds) .joinFtsIdOnly(i, ftsTable) .withSelect(Nel.of(count(i.id).as("num"))) - .changeWhere(c => c && queryCondition(today, q.fix.account.collectiveId, q.cond)) .build .query[Int] .unique + private def searchDistinctOrgCount( + today: LocalDate, + ftsTable: Option[RFtsResult.Table], + folderIds: Option[Set[Ident]] + )(q: Query): ConnectionIO[Int] = + statsItemSelect(q.fix, today, q.cond, folderIds) + .joinFtsIdOnly(i, ftsTable) + .changeWhere(c => c && i.corrOrg.isNotNull) + .withSelect(Nel.of(countDistinct(i.corrOrg).as("num"))) + .build + .query[Int] + .unique + + private def searchDistinctEquipCount( + today: LocalDate, + ftsTable: Option[RFtsResult.Table], + folderIds: Option[Set[Ident]] + )(q: Query): ConnectionIO[Int] = + statsItemSelect(q.fix, today, q.cond, folderIds) + .joinFtsIdOnly(i, ftsTable) + .changeWhere(c => c && i.concEquipment.isNotNull) + .withSelect(Nel.of(countDistinct(i.concEquipment).as("num"))) + .build + .query[Int] + .unique + + private def searchDistinctPersonCount( + today: LocalDate, + ftsTable: Option[RFtsResult.Table], + folderIds: Option[Set[Ident]] + )(q: Query): ConnectionIO[Int] = { + val where = + statsItemWhere(q.fix, today, q.fix.account.collectiveId, q.cond, folderIds) + val corr0: Select = + Select(select(i.corrPerson), from(i), where && i.corrPerson.isNotNull) + val conc0: Select = + Select(select(i.concPerson), from(i), where && i.concPerson.isNotNull) + val corr = + if (ftsTable.isDefined) corr0.joinFtsIdOnly(i, ftsTable) else corr0 + val conc = + if (ftsTable.isDefined) conc0.joinFtsIdOnly(i, ftsTable) else conc0 + val persItem = RItem.as("pers") + Select( + select(countDistinct(persItem.corrPerson).as("num")), + from(union(corr, conc), "pers"), + Condition.unit + ).build.query[Int].unique + } + def searchCorrOrgSummary(today: LocalDate, ftsTable: Option[RFtsResult.Table])( q: Query ): ConnectionIO[List[IdRefCount]] = @@ -460,62 +670,8 @@ object QItem extends FtsSupport { def searchFieldSummary(today: LocalDate, ftsTable: Option[RFtsResult.Table])( q: Query - ): ConnectionIO[List[FieldStats]] = { - val fieldJoin = - from(cv) - .innerJoin(cf, cf.id === cv.field) - .innerJoin(i, i.id === cv.itemId) - - val base = - findItemsBase(q.fix, today, 0, None).unwrap - .changeFrom(_.prepend(fieldJoin)) - .changeWhere(c => c && queryCondition(today, q.fix.account.collectiveId, q.cond)) - .ftsCondition(i, ftsTable) - .groupBy(GroupBy(cf.all)) - - val basicFields = Nel.of( - count(i.id).as("fc"), - const(0).as("favg"), - const(0).as("fsum"), - const(0).as("fmax"), - const(0).as("fmin") - ) - val valueNum = cast(cv.value.s, "decimal").s - val numericFields = Nel.of( - count(i.id).as("fc"), - avg(valueNum).as("favg"), - sum(valueNum).as("fsum"), - max(valueNum).as("fmax"), - min(valueNum).as("fmin") - ) - - val numTypes = Nel.of(CustomFieldType.money, CustomFieldType.numeric) - val query = - union( - base - .withSelect(select(cf.all).concatNel(basicFields)) - .changeWhere(c => c && cf.ftype.notIn(numTypes)), - base - .withSelect(select(cf.all).concatNel(numericFields)) - .changeWhere(c => c && cf.ftype.in(numTypes)) - ).build.query[FieldStats].to[List] - - val fallback = base - .withSelect(select(cf.all).concatNel(basicFields)) - .build - .query[FieldStats] - .to[List] - - query.attemptSql.flatMap { - case Right(res) => res.pure[ConnectionIO] - case Left(ex) => - logger - .error(ex)( - s"Calculating custom field summary failed. You may have invalid custom field values according to their type." - ) *> - fallback - } - } + ): ConnectionIO[List[FieldStats]] = + QItemFieldStats.searchFieldSummary(today, ftsTable)(q) /** Same as `findItems` but resolves the tags for each item. Note that this is * implemented by running an additional query per item. diff --git a/modules/store/src/main/scala/docspell/store/queries/QItemFieldStats.scala b/modules/store/src/main/scala/docspell/store/queries/QItemFieldStats.scala new file mode 100644 index 0000000000..7057a408e1 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QItemFieldStats.scala @@ -0,0 +1,258 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.queries + +import java.time.LocalDate + +import cats.data.{NonEmptyList => Nel} +import cats.implicits._ + +import docspell.common._ +import docspell.store.fts.RFtsResult +import docspell.store.qb.DSL._ +import docspell.store.qb._ +import docspell.store.records._ + +import doobie.implicits._ +import doobie.{Query => _, _} + +private[queries] object QItemFieldStats extends FtsSupport { + private[this] val logger = docspell.logging.getLogger[ConnectionIO] + + private val cf = RCustomField.as("cf") + private val cv = RCustomFieldValue.as("cvf") + private val si = RItem.as("si") + + final case class StatsItemContext( + folderIds: Set[Ident], + matchingSubselect: Select, + matchingItemIds: List[Ident], + matchCount: Int + ) + + /** When the match set is small, use literal `item_id` values so PostgreSQL can use + * `custom_field_value_item_id_idx` instead of nested-looping over a sub-select of + * matching items. + */ + private val fieldStatsItemIdLiteralMax = 200 + + def resolveStatsItemContext( + fix: Query.Fix, + today: LocalDate, + cond: Query.QueryCond, + ftsTable: Option[RFtsResult.Table] + ): ConnectionIO[StatsItemContext] = + for { + folderIds <- QFolder.getMemberFolders(fix.account.collectiveId, fix.account.userId) + folderIdsOpt = Some(folderIds) + matchingSubselect = QItem.statsItemMatchingIdsSelect( + fix, + today, + cond, + ftsTable, + folderIdsOpt + ) + matchCount <- QItem.statsItemMatchCount( + fix, + today, + cond, + ftsTable, + folderIdsOpt + ) + matchingItemIds <- + if (matchCount == 0) Nil.pure[ConnectionIO] + else if (matchCount <= fieldStatsItemIdLiteralMax) + matchingSubselect.build.query[Ident].to[List] + else Nil.pure[ConnectionIO] + } yield StatsItemContext(folderIds, matchingSubselect, matchingItemIds, matchCount) + + def searchFieldDefinitionCountForQuery( + fix: Query.Fix, + today: LocalDate, + cond: Query.QueryCond, + ftsTable: Option[RFtsResult.Table], + folderIds: Set[Ident], + itemCount: Int + ): ConnectionIO[Int] = { + val coll = fix.account.collectiveId + if (itemCount == 0) + 0.pure[ConnectionIO] + else if (itemCount <= fieldStatsItemIdLiteralMax) + resolveStatsItemContext(fix, today, cond, ftsTable).flatMap( + searchFieldDefinitionCount(_)(Query(fix, cond)) + ) + else + fieldDefinitionCountCfFirst(coll, fix, today, cond, ftsTable, folderIds) + } + + /** Count non-numeric custom fields that have at least one value on a matching item. + * Iterates `custom_field` (typically few rows) and uses `EXISTS` with the + * `custom_field_value.field` index instead of `cvf.item_id IN (large subselect)`. + */ + private def fieldDefinitionCountCfFirst( + coll: CollectiveId, + fix: Query.Fix, + today: LocalDate, + cond: Query.QueryCond, + ftsTable: Option[RFtsResult.Table], + folderIds: Set[Ident] + ): ConnectionIO[Int] = + Select( + select(count(cf.id).as("num")), + from(cf), + cf.cid === coll && + cf.ftype.notIn(fieldStatsNumTypes) && + fieldHasMatchingValue(fix, today, coll, cond, ftsTable, folderIds) + ).build.query[Int].unique + + private def fieldHasMatchingValue( + fix: Query.Fix, + today: LocalDate, + coll: CollectiveId, + cond: Query.QueryCond, + ftsTable: Option[RFtsResult.Table], + folderIds: Set[Ident] + ): Condition = { + val probe = Select( + select(const(1)), + from(si).innerJoin(cv, cv.itemId === si.id), + cv.field === cf.id && + QItem.statsItemWhereFor(si, fix, today, coll, cond, Some(folderIds)) + ).joinFtsIdOnly(si, ftsTable).limit(1) + Condition.CompareSelect(const(1), Operator.Eq, probe) + } + + private def cvItemFilter(ctx: StatsItemContext): Option[Condition] = + if (ctx.matchCount == 0) None + else if (ctx.matchingItemIds.nonEmpty) + Some(cvItemIdFilter(ctx.matchingItemIds, ctx.matchingSubselect)) + else + Some(cv.itemId.in(ctx.matchingSubselect)) + + private def cvItemIdFilter( + itemIds: List[Ident], + matchingSubselect: Select + ): Condition = + itemIds match { + case id :: Nil => + cv.itemId === id + case _ => + Nel.fromList(itemIds) match { + case Some(nel) if itemIds.length <= fieldStatsItemIdLiteralMax => + cv.itemId.in(nel) + case _ => + cv.itemId.in(matchingSubselect) + } + } + + private def fieldStatsBase( + coll: CollectiveId, + cvFilter: Condition + ): Select = + Select( + select(cf.all), + from(cv).innerJoin(cf, cf.id === cv.field && cf.cid === coll), + cvFilter + ) + + private val fieldStatsNumTypes = + Nel.of(CustomFieldType.money, CustomFieldType.numeric) + + private val fieldStatsBasicFields: Nel[SelectExpr] = + Nel.of( + count(cv.itemId).as("fc"), + const(0).as("favg"), + const(0).as("fsum"), + const(0).as("fmax"), + const(0).as("fmin") + ) + + private val fieldStatsNumericFields: Nel[SelectExpr] = { + val valueNum = castNumeric(cv.value.s).s + Nel.of( + count(cv.itemId).as("fc"), + avg(valueNum).as("favg"), + sum(valueNum).as("fsum"), + max(valueNum).as("fmax"), + min(valueNum).as("fmin") + ) + } + + def searchFieldSummary(today: LocalDate, ftsTable: Option[RFtsResult.Table])( + q: Query + ): ConnectionIO[List[FieldStats]] = + resolveStatsItemContext(q.fix, today, q.cond, ftsTable).flatMap { ctx => + searchFieldSummaryFromContext(q.fix.account.collectiveId, ctx) + } + + def searchFieldSummaryFromContext( + coll: CollectiveId, + ctx: StatsItemContext + ): ConnectionIO[List[FieldStats]] = + cvItemFilter(ctx) match { + case None => + List.empty[FieldStats].pure[ConnectionIO] + + case Some(cvFilter) => + val largeSet = ctx.matchCount > fieldStatsItemIdLiteralMax + val base = fieldStatsBase(coll, cvFilter).groupBy(GroupBy(cf.all)) + val query = + if (largeSet) { + // Dashboard/search UI only displays numeric fields with sum > 0; skip text + // fields when the match set is too large to aggregate cheaply. + base + .withSelect(select(cf.all).concatNel(fieldStatsNumericFields)) + .changeWhere(c => c && cf.ftype.in(fieldStatsNumTypes)) + .build + .query[FieldStats] + .to[List] + } else { + union( + base + .withSelect(select(cf.all).concatNel(fieldStatsBasicFields)) + .changeWhere(c => c && cf.ftype.notIn(fieldStatsNumTypes)), + base + .withSelect(select(cf.all).concatNel(fieldStatsNumericFields)) + .changeWhere(c => c && cf.ftype.in(fieldStatsNumTypes)) + ).build.query[FieldStats].to[List] + } + + val fallback = + base + .withSelect(select(cf.all).concatNel(fieldStatsBasicFields)) + .build + .query[FieldStats] + .to[List] + + query.attemptSql.flatMap { + case Right(res) => + res.pure[ConnectionIO] + case Left(ex) => + logger + .error(ex)( + s"Calculating custom field summary failed. You may have invalid custom field values according to their type." + ) *> fallback + } + } + + def searchFieldDefinitionCount( + ctx: StatsItemContext + )(q: Query): ConnectionIO[Int] = { + val coll = q.fix.account.collectiveId + cvItemFilter(ctx) match { + case None => + 0.pure[ConnectionIO] + case Some(filter) => + fieldStatsBase(coll, filter) + .changeWhere(c => c && cf.ftype.notIn(fieldStatsNumTypes)) + .withSelect(Nel.of(countDistinct(cf.id).as("num"))) + .build + .query[Int] + .unique + } + } +} diff --git a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala index c6bff38363..af67af63a0 100644 --- a/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala +++ b/modules/store/src/main/scala/docspell/store/queries/SearchSummary.scala @@ -15,7 +15,11 @@ case class SearchSummary( corrOrgs: List[IdRefCount], corrPers: List[IdRefCount], concPers: List[IdRefCount], - concEquip: List[IdRefCount] + concEquip: List[IdRefCount], + fieldCount: Option[Int] = None, + orgCount: Option[Int] = None, + personCount: Option[Int] = None, + equipCount: Option[Int] = None ) { def onlyExisting: SearchSummary = @@ -28,6 +32,10 @@ case class SearchSummary( corrOrgs = corrOrgs.filter(_.count > 0), corrPers = corrPers.filter(_.count > 0), concPers = concPers.filter(_.count > 0), - concEquip = concEquip.filter(_.count > 0) + concEquip = concEquip.filter(_.count > 0), + fieldCount, + orgCount, + personCount, + equipCount ) } diff --git a/modules/store/src/test/scala/docspell/store/queries/SearchStatsTest.scala b/modules/store/src/test/scala/docspell/store/queries/SearchStatsTest.scala new file mode 100644 index 0000000000..1461e4f1ee --- /dev/null +++ b/modules/store/src/test/scala/docspell/store/queries/SearchStatsTest.scala @@ -0,0 +1,156 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.queries + +import java.time.{Instant, LocalDate} + +import cats.effect.IO +import cats.syntax.traverse._ + +import docspell.common._ +import docspell.store._ +import docspell.store.records._ + +class SearchStatsTest extends DatabaseTest { + + override def munitFixtures = h2Memory + + // H2 covers profile logic only; PostgreSQL/MariaDB planner behavior for + // literal item_id filters is not exercised in CI. + + test("searchStats profiles omit heavy aggregates as expected") { + val store = h2Store() + for { + cid <- prepareItems(store) + _ <- prepareCustomFields(store, cid, Ident.unsafe("item-0")) + today <- IO(LocalDate.now()) + account <- store + .transact(QLogin.findAccount(DocspellSystem.account)) + .map(_.get) + q = Query(Query.Fix(account, None, None), Query.QueryExpr(None)) + full <- store.transact(QItem.searchStats(today, None)(q)) + general <- store.transact(QItem.searchStatsGeneral(today, None)(q)) + fields <- store.transact(QItem.searchStatsFields(today, None)(q)) + } yield { + assertEquals(general.fields, Nil) + assertEquals(general.folders, Nil) + assertEquals(general.corrOrgs, Nil) + assert(general.fieldCount.isDefined) + assertEquals(general.fieldCount.get, 1) + assert(general.orgCount.isDefined) + assert(general.personCount.isDefined) + assert(general.equipCount.isDefined) + assertEquals(general.count, 5) + + assertEquals(fields.tags, Nil) + assertEquals(fields.folders, Nil) + assertEquals(fields.count, 5) + assert(fields.fieldCount.isEmpty) + assertEquals(fields.fields.length, 2) + assert(fields.fields.exists(_.field.ftype == CustomFieldType.money)) + + assert(full.fields.length >= 2) + assert(full.count >= 5) + assert(general.count == full.count) + } + } + + private def prepareItems(store: Store[IO]): IO[CollectiveId] = + for { + cid <- store.transact(RCollective.insert(makeCollective)) + _ <- store.transact(RUser.insert(makeUser(cid))) + items = (0 until 5).map(makeItem(_, cid)).toList + _ <- items.traverse(i => store.transact(RItem.insert(i))) + } yield cid + + private def prepareCustomFields(store: Store[IO], cid: CollectiveId, itemId: Ident) = { + val textField = RCustomField( + Ident.unsafe("cf-text"), + Ident.unsafe("notes"), + Some("Notes"), + cid, + CustomFieldType.text, + ts + ) + val moneyField = RCustomField( + Ident.unsafe("cf-money"), + Ident.unsafe("amount"), + Some("Amount"), + cid, + CustomFieldType.money, + ts + ) + store.transact( + for { + _ <- RCustomField.insert(textField) + _ <- RCustomField.insert(moneyField) + _ <- RCustomFieldValue.insert( + RCustomFieldValue( + Ident.unsafe("cv-text"), + itemId, + textField.id, + "hello" + ) + ) + _ <- RCustomFieldValue.insert( + RCustomFieldValue( + Ident.unsafe("cv-money"), + itemId, + moneyField.id, + "10.50" + ) + ) + } yield () + ) + } + + private def makeUser(cid: CollectiveId): RUser = + RUser( + Ident.unsafe("uid1"), + DocspellSystem.account.user, + cid, + Password("test"), + UserState.Active, + AccountSource.Local, + None, + 0, + None, + Timestamp(Instant.now) + ) + + private def makeCollective: RCollective = + RCollective( + CollectiveId.unknown, + DocspellSystem.account.collective, + CollectiveState.Active, + Language.English, + integrationEnabled = true, + ts + ) + + private def makeItem(n: Int, cid: CollectiveId): RItem = + RItem( + Ident.unsafe(s"item-$n"), + cid, + s"item $n", + None, + "test", + Direction.Incoming, + ItemState.Created, + None, + None, + None, + None, + None, + ts, + ts, + None, + None + ) + + private val ts = Timestamp.ofMillis(1654329963743L) +} diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index 3ed94ba06f..efa761e308 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -123,6 +123,9 @@ module Api exposing , itemSearchBookmark , itemSearchStats , itemSearchStatsBookmark + , itemSearchStatsGeneral + , itemSearchStatsFields + , itemSearchStatsBookmarkAt , login , loginSession , logout @@ -2128,10 +2131,13 @@ itemSearchBookmark flags bmSearch receive = |> Task.attempt receive -itemSearchStatsTask : Flags -> ItemQuery -> Task.Task Http.Error SearchStats -itemSearchStatsTask flags search = +itemSearchStatsAt : String -> Flags -> ItemQuery -> Task.Task Http.Error SearchStats +itemSearchStatsAt profile flags search = Http2.authTask - { url = flags.config.baseUrl ++ "/api/v1/sec/item/searchStats" + { url = + flags.config.baseUrl + ++ "/api/v1/sec/item/searchStats/" + ++ profile , method = "POST" , headers = [] , account = getAccount flags @@ -2141,20 +2147,55 @@ itemSearchStatsTask flags search = } +{-| Legacy search stats endpoint (defaults to full profile on the server). +Use `itemSearchStatsAt "full"` for the canonical path URL. +-} +itemSearchStatsTask : Flags -> ItemQuery -> Task.Task Http.Error SearchStats +itemSearchStatsTask flags search = + itemSearchStatsAt "full" flags search + + itemSearchStats : Flags -> ItemQuery -> (Result Http.Error SearchStats -> msg) -> Cmd msg itemSearchStats flags search receive = itemSearchStatsTask flags search |> Task.attempt receive +itemSearchStatsGeneral : Flags -> ItemQuery -> (Result Http.Error SearchStats -> msg) -> Cmd msg +itemSearchStatsGeneral flags search receive = + itemSearchStatsAt "general" flags search |> Task.attempt receive + + +itemSearchStatsFields : Flags -> ItemQuery -> (Result Http.Error SearchStats -> msg) -> Cmd msg +itemSearchStatsFields flags search receive = + itemSearchStatsAt "fields" flags search |> Task.attempt receive + + +itemSearchStatsBookmarkAt : String -> Flags -> ItemQuery -> (Result Http.Error SearchStats -> msg) -> Cmd msg +itemSearchStatsBookmarkAt profile flags search receive = + let + getBookmark = + getBookmarkByIdTask flags search.query + |> Task.map (\bm -> { search | query = bm.query }) + + getStats q = + itemSearchStatsAt profile flags q + in + Task.andThen getStats getBookmark + |> Task.attempt receive + + itemSearchStatsBookmark : Flags -> ItemQuery -> (Result Http.Error SearchStats -> msg) -> Cmd msg itemSearchStatsBookmark flags search receive = let + profile = + Maybe.withDefault "full" search.statsProfile + getBookmark = getBookmarkByIdTask flags search.query |> Task.map (\bm -> { search | query = bm.query }) getStats q = - itemSearchStatsTask flags q + itemSearchStatsAt profile flags q in Task.andThen getStats getBookmark |> Task.attempt receive diff --git a/modules/webapp/src/main/elm/App/Update.elm b/modules/webapp/src/main/elm/App/Update.elm index fc68b26f07..dc7f8a9713 100644 --- a/modules/webapp/src/main/elm/App/Update.elm +++ b/modules/webapp/src/main/elm/App/Update.elm @@ -411,8 +411,12 @@ applyClientSettings texts model settings = , setTheme , Sub.none ) - , updateDashboard texts Page.Dashboard.Data.reloadUiSettings - , updateSearch texts Page.Search.Data.UiSettingsUpdated + , \m -> + if Page.isSearchPage m.page then + updateSearch texts Page.Search.Data.UiSettingsUpdated m + + else + ( m, Cmd.none, Sub.none ) , updateItemDetail texts Page.ItemDetail.Data.UiSettingsUpdated ] { model | uiSettings = settings } @@ -771,4 +775,4 @@ initPage model_ page = ( model, Cmd.none, Sub.none ) DashboardPage -> - ( model, Cmd.map DashboardMsg (Page.Dashboard.Data.reinitCmd model.flags), Sub.none ) + ( model, Cmd.none, Sub.none ) diff --git a/modules/webapp/src/main/elm/Comp/BoxQueryView.elm b/modules/webapp/src/main/elm/Comp/BoxQueryView.elm index 6bce5bf1ef..b66df467ca 100644 --- a/modules/webapp/src/main/elm/Comp/BoxQueryView.elm +++ b/modules/webapp/src/main/elm/Comp/BoxQueryView.elm @@ -230,6 +230,7 @@ mkQuery q meta = , offset = Nothing , searchMode = Just <| Data.SearchMode.asString Data.SearchMode.Normal , withDetails = Just meta.details + , statsProfile = Nothing } diff --git a/modules/webapp/src/main/elm/Comp/BoxStatsView.elm b/modules/webapp/src/main/elm/Comp/BoxStatsView.elm index 794ca4921d..a634fcb9ad 100644 --- a/modules/webapp/src/main/elm/Comp/BoxStatsView.elm +++ b/modules/webapp/src/main/elm/Comp/BoxStatsView.elm @@ -5,7 +5,7 @@ -} -module Comp.BoxStatsView exposing (Model, Msg, init, reloadData, update, view) +module Comp.BoxStatsView exposing (Model, Msg, ViewResult(..), init, reloadData, update, view) import Api import Api.Model.ItemQuery exposing (ItemQuery) @@ -25,6 +25,7 @@ import Util.List type alias Model = { results : ViewResult , meta : StatsData + , parentManaged : Bool } @@ -39,13 +40,23 @@ type Msg | ReloadData -init : Flags -> StatsData -> ( Model, Cmd Msg ) -init flags data = - ( { results = Loading - , meta = data - } - , dataCmd flags data - ) +init : Flags -> StatsData -> Maybe (Result Http.Error SearchStats) -> Bool -> ( Model, Cmd Msg ) +init flags data cached parentManaged = + case cached of + Just (Ok stats) -> + ( { results = Loaded stats, meta = data, parentManaged = parentManaged }, Cmd.none ) + + Just (Err err) -> + ( { results = Failed err, meta = data, parentManaged = parentManaged }, Cmd.none ) + + Nothing -> + if parentManaged then + ( { results = Loading, meta = data, parentManaged = parentManaged }, Cmd.none ) + + else + ( { results = Loading, meta = data, parentManaged = parentManaged } + , dataCmd flags data + ) reloadData : Msg @@ -67,7 +78,11 @@ update flags msg model = ( { model | results = Failed err }, Cmd.none, False ) ReloadData -> - ( model, dataCmd flags model.meta, True ) + if model.parentManaged then + ( model, Cmd.none, True ) + + else + ( model, dataCmd flags model.meta, True ) @@ -120,19 +135,22 @@ viewGeneral texts stats = List.length stats.tagCloud.items fieldCount = - List.length stats.fieldStats + Maybe.withDefault (List.length stats.fieldStats) stats.fieldCount orgCount = - List.length stats.corrOrgStats + Maybe.withDefault (List.length stats.corrOrgStats) stats.orgCount persCount = - (stats.corrPersStats ++ stats.concPersStats) - |> List.map (.ref >> .id) - |> Util.List.distinct - |> List.length + Maybe.withDefault + ((stats.corrPersStats ++ stats.concPersStats) + |> List.map (.ref >> .id) + |> Util.List.distinct + |> List.length + ) + stats.personCount equipCount = - List.length stats.concEquipStats + Maybe.withDefault (List.length stats.concEquipStats) stats.equipCount mklabel name = div [ class "py-0.5 text-lg" ] [ text name ] @@ -166,21 +184,50 @@ viewGeneral texts stats = --- Helpers -mkQuery : String -> ItemQuery -mkQuery query = +mkQuery : String -> Maybe String -> ItemQuery +mkQuery query profile = { query = query , limit = Nothing , offset = Nothing , searchMode = Nothing , withDetails = Nothing + , statsProfile = profile } +statsProfileFor : SummaryShow -> Maybe String +statsProfileFor show = + case show of + SummaryShowGeneral -> + Just "general" + + SummaryShowFields _ -> + Just "fields" + + dataCmd : Flags -> StatsData -> Cmd Msg dataCmd flags data = - case data.query of - SearchQueryString q -> - Api.itemSearchStats flags (mkQuery q) StatsResp + let + profile = + statsProfileFor data.show + + query = + case data.query of + SearchQueryString q -> + mkQuery q profile + + SearchQueryBookmark bmId -> + mkQuery bmId profile + in + case ( data.query, profile ) of + ( SearchQueryString _, Just "general" ) -> + Api.itemSearchStatsGeneral flags query StatsResp + + ( SearchQueryString _, _ ) -> + Api.itemSearchStatsFields flags query StatsResp + + ( SearchQueryBookmark _, Just "general" ) -> + Api.itemSearchStatsBookmarkAt "general" flags query StatsResp - SearchQueryBookmark bmId -> - Api.itemSearchStatsBookmark flags (mkQuery bmId) StatsResp + ( SearchQueryBookmark _, _ ) -> + Api.itemSearchStatsBookmarkAt "fields" flags query StatsResp diff --git a/modules/webapp/src/main/elm/Comp/BoxView.elm b/modules/webapp/src/main/elm/Comp/BoxView.elm index f2c3a3e289..fa055fafe2 100644 --- a/modules/webapp/src/main/elm/Comp/BoxView.elm +++ b/modules/webapp/src/main/elm/Comp/BoxView.elm @@ -8,9 +8,11 @@ module Comp.BoxView exposing (..) import Comp.BoxQueryView -import Comp.BoxStatsView +import Api.Model.SearchStats exposing (SearchStats) +import Comp.BoxStatsView exposing (ViewResult(..)) import Comp.BoxUploadView import Data.Box exposing (Box) +import Http import Data.BoxContent exposing (BoxContent(..), MessageData) import Data.Flags exposing (Flags) import Data.UiSettings exposing (UiSettings) @@ -42,11 +44,11 @@ type Msg | ReloadData -init : Flags -> Box -> ( Model, Cmd Msg ) -init flags box = +init : Flags -> Box -> Maybe (Result Http.Error SearchStats) -> Bool -> ( Model, Cmd Msg ) +init flags box statsCached parentManaged = let ( cm, cc ) = - contentInit flags box.content + contentInit flags box.content statsCached parentManaged in ( { box = box , content = cm @@ -61,8 +63,30 @@ reloadData = ReloadData -contentInit : Flags -> BoxContent -> ( ContentModel, Cmd Msg ) -contentInit flags content = +applyStatsResult : Result Http.Error SearchStats -> Model -> Model +applyStatsResult result model = + case model.content of + ContentStats sm -> + { model + | content = + ContentStats + { sm + | results = + case result of + Ok stats -> + Loaded stats + + Err err -> + Failed err + } + } + + _ -> + model + + +contentInit : Flags -> BoxContent -> Maybe (Result Http.Error SearchStats) -> Bool -> ( ContentModel, Cmd Msg ) +contentInit flags content statsCached parentManaged = case content of BoxMessage data -> ( ContentMessage data, Cmd.none ) @@ -84,7 +108,7 @@ contentInit flags content = BoxStats data -> let ( sm, sc ) = - Comp.BoxStatsView.init flags data + Comp.BoxStatsView.init flags data statsCached parentManaged in ( ContentStats sm, Cmd.map StatsMsg sc ) diff --git a/modules/webapp/src/main/elm/Comp/DashboardView.elm b/modules/webapp/src/main/elm/Comp/DashboardView.elm index 28958ef39a..f78e18e76d 100644 --- a/modules/webapp/src/main/elm/Comp/DashboardView.elm +++ b/modules/webapp/src/main/elm/Comp/DashboardView.elm @@ -5,47 +5,122 @@ -} -module Comp.DashboardView exposing (Model, Msg, init, reloadData, update, view, viewBox) +-- Dashboard coordinates search statistics HTTP requests and caches results in +-- StatsCache. Boxes with parentManaged = True do not fetch stats themselves; +-- this module issues one request per distinct (query, profile) pair via +-- deduplicated fetchStatsCmds. + +module Comp.DashboardView exposing (Model, Msg, emptyModel, init, reload, reloadData, update, view, viewBox) + +import Api +import Api.Model.ItemQuery exposing (ItemQuery) +import Api.Model.SearchStats exposing (SearchStats) import Comp.BoxView +import Data.Box exposing (Box) +import Data.BoxContent exposing (BoxContent(..), SearchQuery(..), StatsData, SummaryShow(..), searchQueryAsString, searchQueryFromString) import Data.Dashboard exposing (Dashboard) import Data.Flags exposing (Flags) import Data.UiSettings exposing (UiSettings) import Dict exposing (Dict) import Html exposing (Html, div) import Html.Attributes exposing (class) +import Http import Messages.Comp.DashboardView exposing (Texts) +import Util.List import Util.Update +type alias StatsKey = + String + + +type alias StatsCache = + { general : Dict StatsKey (Result Http.Error SearchStats) + , fields : Dict StatsKey (Result Http.Error SearchStats) + } + + +emptyStatsCache : StatsCache +emptyStatsCache = + { general = Dict.empty + , fields = Dict.empty + } + + type alias Model = { dashboard : Dashboard , boxModels : Dict Int Comp.BoxView.Model + , statsCache : StatsCache } type Msg = BoxMsg Int Comp.BoxView.Msg | ReloadData + | StatsGeneralResp StatsKey (Result Http.Error SearchStats) + | StatsFieldsResp StatsKey (Result Http.Error SearchStats) + + +emptyModel : Dashboard -> Model +emptyModel db = + { dashboard = db + , boxModels = Dict.empty + , statsCache = emptyStatsCache + } init : Flags -> Dashboard -> ( Model, Cmd Msg ) init flags db = + initWithCache flags db emptyStatsCache + + +initWithCache : Flags -> Dashboard -> StatsCache -> ( Model, Cmd Msg ) +initWithCache flags db cache = let - ( boxModels, cmds ) = - List.map (Comp.BoxView.init flags) db.boxes - |> List.indexedMap (\a -> \( bm, bc ) -> ( bm, Cmd.map (BoxMsg a) bc )) - |> List.unzip + indexedBoxes = + List.indexedMap (initBox flags cache) db.boxes + + boxModels = + List.map (\( index, bm, _ ) -> ( index, bm )) indexedBoxes + + boxCmds = + List.map (\( _, _, cmd ) -> cmd) indexedBoxes + + statsCmds = + fetchStatsCmds flags db cache + in ( { dashboard = db - , boxModels = - List.indexedMap Tuple.pair boxModels - |> Dict.fromList + , boxModels = Dict.fromList boxModels + , statsCache = cache } - , Cmd.batch cmds + , Cmd.batch (statsCmds ++ boxCmds) ) +reload : Flags -> Model -> ( Model, Cmd Msg ) +reload flags model = + initWithCache flags model.dashboard model.statsCache + + +initBox : Flags -> StatsCache -> Int -> Box -> ( Int, Comp.BoxView.Model, Cmd Msg ) +initBox flags cache index box = + let + ( statsCached, parentManaged ) = + case box.content of + BoxStats data -> + ( lookupStats cache data, True ) + + _ -> + ( Nothing, False ) + + ( bm, bc ) = + Comp.BoxView.init flags box statsCached parentManaged + in + ( index, bm, Cmd.map (BoxMsg index) bc ) + + reloadData : Msg reloadData = ReloadData @@ -75,13 +150,42 @@ update flags msg model = ReloadData -> let - updateAll = - List.map (\index -> BoxMsg index Comp.BoxView.reloadData) (Dict.keys model.boxModels) - |> List.map (\m -> update flags m) - |> Util.Update.andThen2 + ( dm, cmds ) = + reload flags model in - updateAll model + ( dm, cmds, Sub.none ) + StatsGeneralResp key result -> + let + sc = + model.statsCache + + cache = + { sc | general = Dict.insert key result sc.general } + + boxes = + updateStatsBoxes cache model.boxModels + in + ( { model | statsCache = cache, boxModels = boxes } + , Cmd.none + , Sub.none + ) + + StatsFieldsResp key result -> + let + sc = + model.statsCache + + cache = + { sc | fields = Dict.insert key result sc.fields } + + boxes = + updateStatsBoxes cache model.boxModels + in + ( { model | statsCache = cache, boxModels = boxes } + , Cmd.none + , Sub.none + ) unit : Model -> ( Model, Cmd Msg, Sub Msg ) unit model = @@ -97,7 +201,9 @@ view texts flags settings model = div [ class (gridStyle model.dashboard) ] - (List.indexedMap (viewBox texts flags settings) <| Dict.values model.boxModels) + (Dict.toList model.boxModels + |> List.map (\( index, box ) -> viewBox texts flags settings index box) + ) viewBox : Texts -> Flags -> UiSettings -> Int -> Comp.BoxView.Model -> Html Msg @@ -110,10 +216,137 @@ viewBox texts flags settings index box = --- Helpers -{-| note due to tailwinds purging css that is not found in source -files, need to spell them out somewhere - which is done it keep.txt in -this case. --} +lookupStats : StatsCache -> StatsData -> Maybe (Result Http.Error SearchStats) +lookupStats cache data = + let + key = + searchQueryAsString data.query + in + case data.show of + SummaryShowGeneral -> + Dict.get key cache.general + + SummaryShowFields _ -> + Dict.get key cache.fields + + +updateStatsBoxes : StatsCache -> Dict Int Comp.BoxView.Model -> Dict Int Comp.BoxView.Model +updateStatsBoxes cache = + Dict.map + (\_ bm -> + case bm.content of + Comp.BoxView.ContentStats sm -> + case lookupStats cache sm.meta of + Just result -> + Comp.BoxView.applyStatsResult result bm + + Nothing -> + bm + + _ -> + bm + ) + + +isCached : StatsCache -> StatsKey -> String -> Bool +isCached cache key profile = + let + cached = + case profile of + "general" -> + Dict.get key cache.general + + _ -> + Dict.get key cache.fields + in + case cached of + Just (Ok _) -> + True + + _ -> + False + + +fetchStatsCmds : Flags -> Dashboard -> StatsCache -> List (Cmd Msg) +fetchStatsCmds flags db cache = + db.boxes + |> List.concatMap statsRequestsForBox + |> Util.List.distinct + |> List.filter (\( k, p ) -> not (isCached cache k p)) + |> List.filterMap (fetchStatsPair flags) + + +statsRequestsForBox : Box -> List ( StatsKey, String ) +statsRequestsForBox box = + case box.content of + BoxStats data -> + let + key = + searchQueryAsString data.query + in + case data.show of + SummaryShowGeneral -> + [ ( key, "general" ) ] + + SummaryShowFields _ -> + [ ( key, "fields" ) ] + + _ -> + [] + + +fetchStatsPair : Flags -> ( StatsKey, String ) -> Maybe (Cmd Msg) +fetchStatsPair flags ( key, profile ) = + searchQueryFromString key + |> Maybe.map + (\sq -> + let + query = + mkStatsQuery sq profile + + resp = + case profile of + "general" -> + StatsGeneralResp key + + _ -> + StatsFieldsResp key + in + case ( sq, profile ) of + ( SearchQueryString _, "general" ) -> + Api.itemSearchStatsGeneral flags query resp + + ( SearchQueryString _, _ ) -> + Api.itemSearchStatsFields flags query resp + + ( SearchQueryBookmark _, "general" ) -> + Api.itemSearchStatsBookmarkAt "general" flags query resp + + ( SearchQueryBookmark _, _ ) -> + Api.itemSearchStatsBookmarkAt "fields" flags query resp + ) + + +mkStatsQuery : SearchQuery -> String -> ItemQuery +mkStatsQuery sq profile = + let + qstr = + case sq of + SearchQueryString s -> + s + + SearchQueryBookmark id -> + id + in + { query = qstr + , limit = Nothing + , offset = Nothing + , searchMode = Nothing + , withDetails = Nothing + , statsProfile = Just profile + } + + gridStyle : Dashboard -> String gridStyle db = let diff --git a/modules/webapp/src/main/elm/Comp/ItemMerge.elm b/modules/webapp/src/main/elm/Comp/ItemMerge.elm index 2d2c813270..0256a0c922 100644 --- a/modules/webapp/src/main/elm/Comp/ItemMerge.elm +++ b/modules/webapp/src/main/elm/Comp/ItemMerge.elm @@ -68,6 +68,7 @@ initQuery flags searchMode query = , withDetails = Just True , searchMode = Just (Data.SearchMode.asString searchMode) , query = Data.ItemQuery.render query + , statsProfile = Nothing } in ( init [], Api.itemSearch flags itemQuery ItemResp ) diff --git a/modules/webapp/src/main/elm/Comp/ItemSearchInput.elm b/modules/webapp/src/main/elm/Comp/ItemSearchInput.elm index 75c60ac4fd..a5507767ac 100644 --- a/modules/webapp/src/main/elm/Comp/ItemSearchInput.elm +++ b/modules/webapp/src/main/elm/Comp/ItemSearchInput.elm @@ -254,6 +254,7 @@ makeSearchCmd flags model addQuery str = , withDetails = Just False , searchMode = Nothing , query = qstr + , statsProfile = Nothing } in if str == Nothing then diff --git a/modules/webapp/src/main/elm/Comp/SearchMenu.elm b/modules/webapp/src/main/elm/Comp/SearchMenu.elm index 31afbf8662..fddd25260d 100644 --- a/modules/webapp/src/main/elm/Comp/SearchMenu.elm +++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm @@ -32,7 +32,7 @@ import Api.Model.EquipmentList exposing (EquipmentList) import Api.Model.FolderStats exposing (FolderStats) import Api.Model.IdName exposing (IdName) import Api.Model.ItemFieldValue exposing (ItemFieldValue) -import Api.Model.ItemQuery exposing (ItemQuery) +import Api.Model.ItemQuery as RQ import Api.Model.PersonList exposing (PersonList) import Api.Model.ReferenceList exposing (ReferenceList) import Api.Model.SearchStats exposing (SearchStats) @@ -116,6 +116,17 @@ type TextSearchModel | Names (Maybe String) +tagsStatsQuery : RQ.ItemQuery +tagsStatsQuery = + { offset = Nothing + , limit = Nothing + , withDetails = Nothing + , searchMode = Nothing + , query = "" + , statsProfile = Just "general" + } + + init : Flags -> Model init flags = { tagSelectModel = Comp.TagSelect.init [] [] @@ -541,7 +552,7 @@ updateDrop ddm flags settings msg model = { model = mdp , cmd = Cmd.batch - [ Api.itemSearchStats flags Api.Model.ItemQuery.empty GetAllTagsResp + [ Api.itemSearchStatsGeneral flags tagsStatsQuery GetAllTagsResp , Api.getOrgLight flags GetOrgResp , Api.getEquipments flags "" Data.EquipmentOrder.NameAsc GetEquipResp , Api.getPersons flags "" Data.PersonOrder.NameAsc GetPersonResp @@ -557,7 +568,7 @@ updateDrop ddm flags settings msg model = ResetForm -> { model = resetModel model - , cmd = Api.itemSearchStats flags Api.Model.ItemQuery.empty GetAllTagsResp + , cmd = Api.itemSearchStatsGeneral flags tagsStatsQuery GetAllTagsResp , sub = Sub.none , stateChange = True , dragDrop = DD.DragDropData ddm Nothing diff --git a/modules/webapp/src/main/elm/Data/BoxContent.elm b/modules/webapp/src/main/elm/Data/BoxContent.elm index f563096923..056e442518 100644 --- a/modules/webapp/src/main/elm/Data/BoxContent.elm +++ b/modules/webapp/src/main/elm/Data/BoxContent.elm @@ -20,6 +20,8 @@ module Data.BoxContent exposing , emptyQueryData , emptyStatsData , emptyUploadData + , searchQueryAsString + , searchQueryFromString ) import Data.ItemColumn exposing (ItemColumn) diff --git a/modules/webapp/src/main/elm/Data/ItemQuery.elm b/modules/webapp/src/main/elm/Data/ItemQuery.elm index 5feff47687..690c7e12c6 100644 --- a/modules/webapp/src/main/elm/Data/ItemQuery.elm +++ b/modules/webapp/src/main/elm/Data/ItemQuery.elm @@ -83,6 +83,7 @@ request smode mq = , withDetails = Just True , query = renderMaybe mq , searchMode = Data.SearchMode.asString smode |> Just + , statsProfile = Nothing } diff --git a/modules/webapp/src/main/elm/Page/Dashboard/Data.elm b/modules/webapp/src/main/elm/Page/Dashboard/Data.elm index 228177382e..fdea94f19a 100644 --- a/modules/webapp/src/main/elm/Page/Dashboard/Data.elm +++ b/modules/webapp/src/main/elm/Page/Dashboard/Data.elm @@ -91,22 +91,15 @@ type Msg init : Flags -> ( Model, Cmd Msg ) init flags = - let - ( dm, dc ) = - Comp.DashboardView.init flags Data.Dashboard.empty - in ( { sideMenu = { bookmarkChooser = Comp.BookmarkChooser.init Data.Bookmarks.empty } - , content = Home dm + , content = Home (Comp.DashboardView.emptyModel Data.Dashboard.empty) , pageError = Nothing , dashboards = Data.Dashboards.emptyAll , isPredefined = True } - , Cmd.batch - [ initCmd flags - , Cmd.map DashboardMsg dc - ] + , initCmd flags ) diff --git a/modules/webapp/src/main/elm/Page/Dashboard/Update.elm b/modules/webapp/src/main/elm/Page/Dashboard/Update.elm index a4e1072b77..3d0c028cf7 100644 --- a/modules/webapp/src/main/elm/Page/Dashboard/Update.elm +++ b/modules/webapp/src/main/elm/Page/Dashboard/Update.elm @@ -110,9 +110,12 @@ update texts settings navKey flags msg model = dm.dashboard ( dm_, dc ) = - Comp.DashboardView.init flags board + Comp.DashboardView.reload flags dm in - ( { model | content = Home dm_ }, Cmd.map DashboardMsg dc, Sub.none ) + ( { model | content = Home dm_ } + , Cmd.map DashboardMsg dc + , Sub.none + ) _ -> unit model diff --git a/modules/webapp/src/main/elm/Page/Share/Update.elm b/modules/webapp/src/main/elm/Page/Share/Update.elm index 12765501e9..678e86e32e 100644 --- a/modules/webapp/src/main/elm/Page/Share/Update.elm +++ b/modules/webapp/src/main/elm/Page/Share/Update.elm @@ -425,6 +425,7 @@ makeSearchCmd flags doInit addResults model = , withDetails = Just True , query = Q.renderMaybe mq , searchMode = Just (Data.SearchMode.asString Data.SearchMode.Normal) + , statsProfile = Nothing } searchCmd =