diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/WorkspaceSpanMetricRequest.java b/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/WorkspaceSpanMetricRequest.java new file mode 100644 index 00000000000..d3b5248e12f --- /dev/null +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/metrics/WorkspaceSpanMetricRequest.java @@ -0,0 +1,45 @@ +package com.comet.opik.api.metrics; + +import com.comet.opik.api.TimeInterval; +import com.comet.opik.api.filter.SpanFilter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.NotNull; +import lombok.Builder; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +/** + * Request for workspace-level span metrics aggregated across projects. When {@code projectIds} is empty, all projects + * in the workspace are included; otherwise only the given projects. {@code intervalEnd} is optional and defaults to + * "now" server-side, mirroring the per-project metrics endpoint. + */ +@Builder(toBuilder = true) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) +public record WorkspaceSpanMetricRequest( + Set<@NotNull UUID> projectIds, + MetricType metricType, + TimeInterval interval, + @Valid BreakdownConfig breakdown, + List filters, + @NotNull Instant intervalStart, + Instant intervalEnd) { + + @AssertTrue(message = "intervalStart must be before intervalEnd") public boolean isStartBeforeEnd() { + return intervalEnd == null || intervalStart.isBefore(intervalEnd); + } + + public boolean hasBreakdown() { + return Optional.ofNullable(breakdown) + .map(BreakdownQueryBuilder::isEnabled) + .orElse(false); + } +} diff --git a/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/WorkspacesResource.java b/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/WorkspacesResource.java index 74fc661394f..f239a73b92f 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/WorkspacesResource.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/WorkspacesResource.java @@ -8,6 +8,7 @@ import com.comet.opik.api.metrics.WorkspaceMetricResponse; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryRequest; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryResponse; +import com.comet.opik.api.metrics.WorkspaceSpanMetricRequest; import com.comet.opik.domain.WorkspaceConfigurationService; import com.comet.opik.domain.WorkspaceMetricsService; import com.comet.opik.domain.workspaces.WorkspaceVersionService; @@ -150,6 +151,29 @@ public Response getCost( return Response.ok().entity(response).build(); } + @POST + @Path("/metrics/spans") + @Operation(operationId = "getWorkspaceSpanMetric", summary = "Get workspace span metric", description = "Gets a span metric time series aggregated across the workspace. When project_ids is empty, all projects in the workspace are included; otherwise only the given projects.", responses = { + @ApiResponse(responseCode = "200", description = "Workspace span metric", content = @Content(schema = @Schema(implementation = WorkspaceMetricResponse.class))), + @ApiResponse(responseCode = "400", description = "Bad Request", content = @Content(schema = @Schema(implementation = ErrorMessage.class))) + }) + @RequiredPermissions(WorkspaceUserPermission.PROJECT_DATA_VIEW) + public Response getWorkspaceSpanMetric( + @RequestBody(content = @Content(schema = @Schema(implementation = WorkspaceSpanMetricRequest.class))) @NotNull @Valid WorkspaceSpanMetricRequest request) { + + String workspaceId = requestContext.get().getWorkspaceId(); + + log.info("Retrieve workspace span metric '{}' for projectIds '{}', on workspace_id '{}'", request.metricType(), + request.projectIds(), workspaceId); + WorkspaceMetricResponse response = workspaceMetricsService.getWorkspaceSpanMetric(request) + .contextWrite(ctx -> setRequestContext(ctx, requestContext)) + .block(); + log.info("Retrieved workspace span metric '{}' for projectIds '{}', on workspace_id '{}'", request.metricType(), + request.projectIds(), workspaceId); + + return Response.ok().entity(response).build(); + } + @GET @Path("/versions") @Operation(operationId = "getWorkspaceVersion", summary = "Get workspace version", description = """ diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsDAO.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsDAO.java index 5596d87e960..df21546d5a0 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsDAO.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsDAO.java @@ -1,11 +1,19 @@ package com.comet.opik.domain; import com.comet.opik.api.DataPoint; +import com.comet.opik.api.InstantToUUIDMapper; +import com.comet.opik.api.TimeInterval; +import com.comet.opik.api.metrics.BreakdownField; +import com.comet.opik.api.metrics.MetricType; import com.comet.opik.api.metrics.WorkspaceMetricRequest; import com.comet.opik.api.metrics.WorkspaceMetricResponse; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryRequest; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryResponse; +import com.comet.opik.api.metrics.WorkspaceSpanMetricRequest; +import com.comet.opik.domain.filter.FilterQueryBuilder; +import com.comet.opik.domain.filter.FilterStrategy; import com.comet.opik.infrastructure.db.TransactionTemplateAsync; +import com.comet.opik.infrastructure.instrumentation.InstrumentAsyncUtils; import com.comet.opik.utils.SentinelTranslation; import com.comet.opik.utils.template.TemplateUtils; import com.google.common.base.Preconditions; @@ -25,18 +33,29 @@ import java.time.Duration; import java.time.Instant; +import java.time.LocalDateTime; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; +import static com.comet.opik.api.metrics.BreakdownQueryBuilder.getBreakdownGroupExpression; import static com.comet.opik.domain.AsyncContextUtils.bindWorkspaceIdToMono; +import static com.comet.opik.infrastructure.FilterUtils.getSTWithLogComment; +import static com.comet.opik.infrastructure.instrumentation.InstrumentAsyncUtils.endSegment; +import static com.comet.opik.infrastructure.instrumentation.InstrumentAsyncUtils.startSegment; import static com.comet.opik.utils.AsyncUtils.makeMonoContextAware; @ImplementedBy(WorkspaceMetricsDAOImpl.class) public interface WorkspaceMetricsDAO { + Set SUPPORTED_SPAN_METRICS = EnumSet.of(MetricType.SPAN_TOKEN_USAGE); + @Deprecated Mono> getFeedbackScoresSummary(WorkspaceMetricsSummaryRequest request); @@ -46,6 +65,8 @@ public interface WorkspaceMetricsDAO { Mono getCostsSummary(WorkspaceMetricsSummaryRequest request); Mono> getCostsDaily(WorkspaceMetricRequest request); + + Mono> getSpanTokenUsage(WorkspaceSpanMetricRequest request); } @Slf4j @@ -209,8 +230,207 @@ STEP toIntervalDay(1) ; """; + // Copied verbatim from ProjectMetricsDAO.SPAN_FILTERED_PREFIX. The only change is the project predicate: + // the per-project `AND project_id = :project_id` becomes an optional IN-list so the same query can span a set + // of projects (or, when project_ids is absent, every project in the workspace). workspace_id stays mandatory. + private static final String SPAN_FILTERED_PREFIX = """ + WITH feedback_scores_deduped AS ( + SELECT workspace_id, + project_id, + entity_id, + name, + value, + last_updated_at, + author, + source_queue_id + FROM ( + SELECT workspace_id, + project_id, + entity_id, + name, + value, + last_updated_at, + last_updated_by AS author, + CAST('' AS FixedString(36)) AS source_queue_id + FROM feedback_scores + WHERE entity_type = 'span' + AND workspace_id = :workspace_id + AND project_id IN :project_ids + AND entity_id >= :uuid_from_time + AND entity_id \\<= :uuid_to_time + UNION ALL + SELECT workspace_id, + project_id, + entity_id, + name, + value, + last_updated_at, + author, + source_queue_id + FROM authored_feedback_scores + WHERE entity_type = 'span' + AND workspace_id = :workspace_id + AND project_id IN :project_ids + AND entity_id >= :uuid_from_time + AND entity_id \\<= :uuid_to_time + ) + ORDER BY last_updated_at DESC + LIMIT 1 BY workspace_id, project_id, entity_id, name, author, source_queue_id + ), feedback_scores_final AS ( + SELECT + workspace_id, + project_id, + entity_id, + name, + if(count() = 1, any(value), toDecimal64(avg(value), 9)) AS value, + max(last_updated_at) AS last_updated_at + FROM feedback_scores_deduped + GROUP BY workspace_id, project_id, entity_id, name + ), + + fsc AS (SELECT entity_id, COUNT(entity_id) AS feedback_scores_count + FROM ( + SELECT * + FROM feedback_scores_final + ORDER BY (workspace_id, project_id, entity_id, name) DESC, last_updated_at DESC + LIMIT 1 BY entity_id, name + ) + GROUP BY entity_id + HAVING + ), + + spans_filtered AS ( + SELECT + id, + UUIDv7ToDateTime(toUUID(id)) as span_time, + duration, + usage, + error_info, + total_estimated_cost + , + project_id, + name, + tags, + metadata, + model, + provider, + type + + FROM ( + SELECT + id, + duration, + usage, + error_info, + total_estimated_cost + , + project_id, + name, + tags, + metadata, + model, + provider, + type + + FROM spans FINAL + + LEFT JOIN fsc ON fsc.entity_id = spans.id + + WHERE workspace_id = :workspace_id + AND project_id IN :project_ids + AND id >= :uuid_from_time + AND id \\<= :uuid_to_time + AND + + AND id in ( + SELECT + entity_id + FROM ( + SELECT * + FROM feedback_scores_final + ORDER BY (workspace_id, project_id, entity_id, name) DESC, last_updated_at DESC + LIMIT 1 BY entity_id, name + ) + GROUP BY entity_id + HAVING + ) + + + AND fsc.feedback_scores_count = 0 + + ) AS t + ) + """; + + // Span filtering is reused from ProjectMetricsDAO's SPAN_FILTERED_PREFIX (above), but the output is shaped in the + // workspace-native style like GET_COSTS_DAILY: each row is a finished series {project_id, name, data}, where data is + // a groupArray(tuple(bucket, value)). No breakdown => one series per usage key; with a provider/model breakdown => + // one series per group, mirroring how GET_COSTS_DAILY_BY_PROJECT groups by project. + private static final String GET_SPAN_TOKEN_USAGE = """ + %s, spans_usage AS ( + SELECT span_time, + name, + value + FROM spans_filtered s + ARRAY JOIN mapKeys(usage) AS name, mapValues(usage) AS value + WHERE value > 0 + ) + , series AS ( + SELECT AS bucket, + name, + sum(value) AS value + FROM spans_usage + GROUP BY name, bucket + ORDER BY name, bucket + WITH FILL + FROM + TO toDateTime(UUIDv7ToDateTime(toUUID(:uuid_to_time))) + STEP + ) + SELECT NULL AS project_id, + name, + groupArray(tuple(bucket, value)) AS data + FROM series + GROUP BY name + SETTINGS log_comment = ''; + """.formatted(SPAN_FILTERED_PREFIX); + + private static final String GET_SPAN_TOKEN_USAGE_WITH_BREAKDOWN = """ + %s, spans_usage AS ( + SELECT span_time, + AS group_name, + value + FROM spans_filtered s + ARRAY JOIN mapKeys(usage) AS name, mapValues(usage) AS value + WHERE value > 0 + AND name = :sub_metric + ) + , series AS ( + SELECT AS bucket, + group_name, + sum(value) AS value + FROM spans_usage + GROUP BY group_name, bucket + ORDER BY group_name, bucket + ) + SELECT NULL AS project_id, + group_name AS name, + groupArray(tuple(bucket, value)) AS data + FROM series + GROUP BY group_name + SETTINGS log_comment = ''; + """.formatted(SPAN_FILTERED_PREFIX); + + private static final String WORKSPACE_METRIC_QUERY_NAME_PREFIX = "WorkspaceMetrics_"; + + private static final Map INTERVAL_TO_SQL = Map.of( + TimeInterval.WEEKLY, "toIntervalWeek(1)", + TimeInterval.DAILY, "toIntervalDay(1)", + TimeInterval.HOURLY, "toIntervalHour(1)"); + private final @NonNull TransactionTemplateAsync template; private final @NonNull IdGenerator idGenerator; + private final @NonNull InstantToUUIDMapper instantToUUIDMapper; @Override public Mono> getFeedbackScoresSummary( @@ -247,6 +467,108 @@ public Mono> getCostsDaily(@NonNull Workspa return getMetricsDaily(request, query); } + @Override + public Mono> getSpanTokenUsage(@NonNull WorkspaceSpanMetricRequest request) { + return template.nonTransaction(connection -> getSpanMetric(request, connection, + request.hasBreakdown() ? GET_SPAN_TOKEN_USAGE_WITH_BREAKDOWN : GET_SPAN_TOKEN_USAGE, + "workspaceSpanTokenUsage") + .flatMapMany(this::rowToDataPoint) + .collectList()); + } + + private Mono getSpanMetric(WorkspaceSpanMetricRequest request, Connection connection, + String query, String segmentName) { + return makeMonoContextAware((userName, workspaceId) -> { + var interval = request.interval(); + var isTotal = interval == TimeInterval.TOTAL; + var hasProjectIds = CollectionUtils.isNotEmpty(request.projectIds()); + + var stTemplate = getSTWithLogComment(query, WORKSPACE_METRIC_QUERY_NAME_PREFIX + segmentName, workspaceId, + userName, hasProjectIds ? request.projectIds().size() : "all"); + + if (isTotal) { + stTemplate.add("bucket", "toDateTime(UUIDv7ToDateTime(toUUID(:uuid_from_time)))"); + } else { + stTemplate.add("step", intervalToSql(interval)) + .add("bucket", wrapWeekly(interval, + "toStartOfInterval(span_time, %s)".formatted(intervalToSql(interval)))) + .add("fill_from", wrapWeekly(interval, + "toStartOfInterval(UUIDv7ToDateTime(toUUID(:uuid_from_time)), %s)" + .formatted(intervalToSql(interval)))); + } + + if (request.hasBreakdown()) { + stTemplate.add("group_expression", + getBreakdownGroupExpression(request.metricType(), request.breakdown())); + } + + Optional.ofNullable(request.filters()) + .ifPresent(filters -> { + FilterQueryBuilder.toAnalyticsDbFilters(filters, FilterStrategy.SPAN) + .ifPresent(spanFilters -> stTemplate.add("span_filters", spanFilters)); + FilterQueryBuilder.toAnalyticsDbFilters(filters, FilterStrategy.SPAN_FEEDBACK_SCORES) + .ifPresent(scoresFilters -> stTemplate.add("span_feedback_scores_filters", + scoresFilters)); + FilterQueryBuilder.toAnalyticsDbFilters(filters, FilterStrategy.SPAN_FEEDBACK_SCORES_IS_EMPTY) + .ifPresent(emptyFilters -> stTemplate.add("feedback_scores_empty_filters", + emptyFilters)); + }); + + if (hasProjectIds) { + stTemplate.add("project_ids", true); + } + + stTemplate.add("uuid_from_time", true); + stTemplate.add("uuid_to_time", true); + if (!isTotal) { + stTemplate.add("with_fill", true); + } + + var intervalEnd = request.intervalEnd() != null ? request.intervalEnd() : Instant.now(); + var statement = connection.createStatement(stTemplate.render()) + .bind("uuid_from_time", instantToUUIDMapper.toLowerBound(request.intervalStart()).toString()) + .bind("uuid_to_time", instantToUUIDMapper.toUpperBound(intervalEnd).toString()) + .bind("workspace_id", workspaceId); + + if (hasProjectIds) { + statement.bind("project_ids", request.projectIds().toArray(new UUID[0])); + } + + if (request.hasBreakdown() && request.breakdown().field() == BreakdownField.METADATA) { + statement.bind("metadata_key", request.breakdown().metadataKey()); + } + + // SPAN_TOKEN_USAGE breakdown sums a single token-usage entry, selected by its name (sub_metric) + if (request.hasBreakdown() && request.metricType() == MetricType.SPAN_TOKEN_USAGE) { + statement.bind("sub_metric", Optional.ofNullable(request.breakdown().subMetric()).orElse("")); + } + + Optional.ofNullable(request.filters()) + .ifPresent(filters -> { + FilterQueryBuilder.bind(statement, filters, FilterStrategy.SPAN); + FilterQueryBuilder.bind(statement, filters, FilterStrategy.SPAN_FEEDBACK_SCORES); + FilterQueryBuilder.bind(statement, filters, FilterStrategy.SPAN_FEEDBACK_SCORES_IS_EMPTY); + }); + + InstrumentAsyncUtils.Segment segment = startSegment(segmentName, "Clickhouse", "get"); + + return Mono.from(statement.execute()) + .doFinally(signalType -> endSegment(segment)); + }); + } + + private String wrapWeekly(TimeInterval interval, String stmt) { + if (interval == TimeInterval.WEEKLY) { + return "toDateTime(%s)".formatted(stmt); + } + return stmt; + } + + private String intervalToSql(TimeInterval interval) { + return Optional.ofNullable(INTERVAL_TO_SQL.get(interval)) + .orElseThrow(() -> new IllegalArgumentException("Invalid interval: " + interval)); + } + private Mono> getMetricsDaily(WorkspaceMetricRequest request, String query) { return template.nonTransaction(connection -> getMetricsDaily(connection, request, query) @@ -335,7 +657,7 @@ private List> getDailyData(List[] dataArray) { var dataItems = Arrays.stream(dataArray) .filter(CollectionUtils::isNotEmpty) .map(dataItem -> DataPoint.builder() - .time(((OffsetDateTime) dataItem.get(0)).toInstant()) + .time(toInstant(dataItem.get(0))) .value(Optional.ofNullable(dataItem.get(1)).map(Object::toString) .map(Double::parseDouble) .orElse(null)) @@ -345,6 +667,17 @@ private List> getDailyData(List[] dataArray) { return dataItems.isEmpty() ? null : dataItems; } + // Bucket timestamps come back as OffsetDateTime for DateTime64 columns (e.g. the cost query's start_time) but as + // LocalDateTime for plain DateTime expressions (e.g. UUIDv7ToDateTime-derived span_time). Both represent UTC. + private Instant toInstant(Object bucket) { + return switch (bucket) { + case OffsetDateTime offsetDateTime -> offsetDateTime.toInstant(); + case LocalDateTime localDateTime -> localDateTime.toInstant(ZoneOffset.UTC); + default -> throw new IllegalStateException( + "Unexpected bucket time type: " + bucket.getClass().getName()); + }; + } + Double filterNan(Double value) { return SentinelTranslation.nanToNull(value); } diff --git a/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsService.java b/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsService.java index 7bba18ffff4..d6dd03f144f 100644 --- a/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsService.java +++ b/apps/opik-backend/src/main/java/com/comet/opik/domain/WorkspaceMetricsService.java @@ -1,17 +1,24 @@ package com.comet.opik.domain; +import com.comet.opik.api.metrics.BreakdownQueryBuilder; +import com.comet.opik.api.metrics.MetricType; import com.comet.opik.api.metrics.WorkspaceMetricRequest; import com.comet.opik.api.metrics.WorkspaceMetricResponse; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryRequest; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryResponse; +import com.comet.opik.api.metrics.WorkspaceSpanMetricRequest; import com.google.inject.ImplementedBy; import jakarta.inject.Inject; import jakarta.inject.Singleton; +import jakarta.ws.rs.BadRequestException; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import reactor.core.publisher.Mono; +import java.util.List; + @ImplementedBy(WorkspaceMetricsServiceImpl.class) public interface WorkspaceMetricsService { @Deprecated @@ -23,6 +30,8 @@ public interface WorkspaceMetricsService { Mono getWorkspaceCostsSummary(WorkspaceMetricsSummaryRequest request); Mono getWorkspaceCosts(WorkspaceMetricRequest request); + + Mono getWorkspaceSpanMetric(WorkspaceSpanMetricRequest request); } @Slf4j @@ -62,4 +71,45 @@ public Mono getWorkspaceCosts(@NonNull WorkspaceMetricR .results(results) .build()); } + + @Override + public Mono getWorkspaceSpanMetric(@NonNull WorkspaceSpanMetricRequest request) { + validate(request); + return dispatch(request) + .map(results -> WorkspaceMetricResponse.builder() + .results(results) + .build()); + } + + private Mono> dispatch(WorkspaceSpanMetricRequest request) { + return switch (request.metricType()) { + case SPAN_TOKEN_USAGE -> workspaceMetricsDAO.getSpanTokenUsage(request); + default -> throw new BadRequestException("Unsupported metric type '%s'".formatted(request.metricType())); + }; + } + + private void validate(WorkspaceSpanMetricRequest request) { + if (request.metricType() == null) { + throw new BadRequestException("'metric_type' must be provided"); + } + if (request.interval() == null) { + throw new BadRequestException("'interval' must be provided"); + } + if (!WorkspaceMetricsDAO.SUPPORTED_SPAN_METRICS.contains(request.metricType())) { + throw new BadRequestException("Unsupported metric type '%s'. Supported: %s" + .formatted(request.metricType(), WorkspaceMetricsDAO.SUPPORTED_SPAN_METRICS)); + } + if (request.hasBreakdown()) { + try { + BreakdownQueryBuilder.validate(request.breakdown(), request.metricType()); + } catch (IllegalArgumentException exception) { + throw new BadRequestException(exception.getMessage()); + } + if (request.metricType() == MetricType.SPAN_TOKEN_USAGE + && StringUtils.isBlank(request.breakdown().subMetric())) { + throw new BadRequestException( + "'sub_metric' is required for token usage breakdown. It should be the usage key name (e.g., completion_tokens, prompt_tokens)."); + } + } + } } diff --git a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/WorkspaceResourceClient.java b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/WorkspaceResourceClient.java index 328272aa571..08ee8adbe88 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/WorkspaceResourceClient.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/utils/resources/WorkspaceResourceClient.java @@ -6,6 +6,7 @@ import com.comet.opik.api.metrics.WorkspaceMetricResponse; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryRequest; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryResponse; +import com.comet.opik.api.metrics.WorkspaceSpanMetricRequest; import com.comet.opik.infrastructure.auth.RequestContext; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.HttpHeaders; @@ -86,6 +87,26 @@ public WorkspaceMetricResponse getCostsDaily(WorkspaceMetricRequest request, Str } } + public WorkspaceMetricResponse getWorkspaceSpanMetric(WorkspaceSpanMetricRequest request, String apiKey, + String workspaceName) { + try (var response = callGetWorkspaceSpanMetric(request, apiKey, workspaceName)) { + + assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK); + + return response.readEntity(WorkspaceMetricResponse.class); + } + } + + public Response callGetWorkspaceSpanMetric(WorkspaceSpanMetricRequest request, String apiKey, + String workspaceName) { + return client.target(RESOURCE_PATH.formatted(baseURI)) + .path("/metrics/spans") + .request() + .header(HttpHeaders.AUTHORIZATION, apiKey) + .header(RequestContext.WORKSPACE_HEADER, workspaceName) + .post(Entity.json(request)); + } + public void upsertWorkspaceConfiguration(WorkspaceConfiguration configuration, String apiKey, String workspaceName) { try (var response = client.target(RESOURCE_PATH.formatted(baseURI)) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/WorkspacesResourceTest.java b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/WorkspacesResourceTest.java index 1e473141e43..54c5069426f 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/WorkspacesResourceTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/WorkspacesResourceTest.java @@ -3,13 +3,23 @@ import com.comet.opik.api.DataPoint; import com.comet.opik.api.FeedbackScoreItem; import com.comet.opik.api.Span; +import com.comet.opik.api.TimeInterval; import com.comet.opik.api.Trace; import com.comet.opik.api.WorkspaceConfiguration; import com.comet.opik.api.error.ErrorMessage; +import com.comet.opik.api.filter.Operator; +import com.comet.opik.api.filter.SpanField; +import com.comet.opik.api.filter.SpanFilter; +import com.comet.opik.api.metrics.BreakdownConfig; +import com.comet.opik.api.metrics.BreakdownField; +import com.comet.opik.api.metrics.MetricType; +import com.comet.opik.api.metrics.ProjectMetricRequest; +import com.comet.opik.api.metrics.ProjectMetricResponse; import com.comet.opik.api.metrics.WorkspaceMetricRequest; import com.comet.opik.api.metrics.WorkspaceMetricResponse; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryRequest; import com.comet.opik.api.metrics.WorkspaceMetricsSummaryResponse; +import com.comet.opik.api.metrics.WorkspaceSpanMetricRequest; import com.comet.opik.api.resources.utils.AuthTestUtils; import com.comet.opik.api.resources.utils.ClickHouseContainerUtils; import com.comet.opik.api.resources.utils.ClientSupportUtils; @@ -20,6 +30,7 @@ import com.comet.opik.api.resources.utils.TestDropwizardAppExtensionUtils; import com.comet.opik.api.resources.utils.TestUtils; import com.comet.opik.api.resources.utils.WireMockUtils; +import com.comet.opik.api.resources.utils.resources.ProjectMetricsResourceClient; import com.comet.opik.api.resources.utils.resources.ProjectResourceClient; import com.comet.opik.api.resources.utils.resources.SpanResourceClient; import com.comet.opik.api.resources.utils.resources.TraceResourceClient; @@ -47,7 +58,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.ValueSource; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.GenericContainer; @@ -65,6 +78,7 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -81,6 +95,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @DisplayName("Workspace Metrics Resource Test") @@ -124,6 +139,7 @@ class WorkspacesResourceTest { private TraceResourceClient traceResourceClient; private SpanResourceClient spanResourceClient; private WorkspaceResourceClient workspaceResourceClient; + private ProjectMetricsResourceClient projectMetricsResourceClient; private IdGenerator idGenerator; @BeforeAll @@ -135,6 +151,7 @@ void setUpAll(ClientSupport client, IdGenerator idGenerator) throws SQLException this.traceResourceClient = new TraceResourceClient(client, baseURI); this.spanResourceClient = new SpanResourceClient(client, baseURI); this.workspaceResourceClient = new WorkspaceResourceClient(client, baseURI, factory); + this.projectMetricsResourceClient = new ProjectMetricsResourceClient(client, baseURI); this.idGenerator = idGenerator; ClientSupportUtils.config(client); @@ -516,6 +533,295 @@ void costsDaily_emptyData(boolean withProjectIds) { } } + @Nested + @DisplayName("Span metrics") + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SpanMetricsTest { + + @Test + void spanTokenUsage_matchesPerProjectEndpoint() { + var workspaceName = UUID.randomUUID().toString(); + var apiKey = UUID.randomUUID().toString(); + mockTargetWorkspace(apiKey, workspaceName, UUID.randomUUID().toString()); + + String projectName = RandomStringUtils.randomAlphabetic(10); + var projectId = projectResourceClient.createProject(projectName, apiKey, workspaceName); + + createSpansWithUsage(projectName, apiKey, workspaceName, Instant.now(), "completion_tokens", + List.of("openai", "anthropic", "openai")); + + var endTime = Instant.now(); + var startTime = endTime.minus(Duration.ofDays(1)); + + var perProject = projectMetricsResourceClient.getProjectMetrics(projectId, + ProjectMetricRequest.builder() + .metricType(MetricType.SPAN_TOKEN_USAGE) + .interval(TimeInterval.HOURLY) + .intervalStart(startTime) + .intervalEnd(endTime) + .build(), + Number.class, apiKey, workspaceName); + + // Explicit project subset must match the per-project endpoint exactly (the "exact copy" contract) + var workspaceSubset = workspaceResourceClient.getWorkspaceSpanMetric( + spanRequest(MetricType.SPAN_TOKEN_USAGE, Set.of(projectId), startTime, endTime, null), apiKey, + workspaceName); + assertSeriesMatch(seriesFromWorkspace(workspaceSubset.results()), + seriesFromProject(perProject.results())); + + // No project ids => all projects. Workspace has only this project, so it must also match. + var workspaceAll = workspaceResourceClient.getWorkspaceSpanMetric( + spanRequest(MetricType.SPAN_TOKEN_USAGE, null, startTime, endTime, null), apiKey, workspaceName); + assertSeriesMatch(seriesFromWorkspace(workspaceAll.results()), seriesFromProject(perProject.results())); + } + + @Test + void spanTokenUsage_withProviderBreakdown_matchesPerProjectEndpoint() { + var workspaceName = UUID.randomUUID().toString(); + var apiKey = UUID.randomUUID().toString(); + mockTargetWorkspace(apiKey, workspaceName, UUID.randomUUID().toString()); + + String projectName = RandomStringUtils.randomAlphabetic(10); + var projectId = projectResourceClient.createProject(projectName, apiKey, workspaceName); + + // Seed spans with a known usage key and providers so the token-usage breakdown is deterministic + createSpansWithUsage(projectName, apiKey, workspaceName, Instant.now(), "completion_tokens", + List.of("openai", "anthropic", "openai")); + + var endTime = Instant.now(); + var startTime = endTime.minus(Duration.ofDays(1)); + var breakdown = BreakdownConfig.builder() + .field(BreakdownField.PROVIDER) + .subMetric("completion_tokens") + .build(); + + var perProject = projectMetricsResourceClient.getProjectMetrics(projectId, + ProjectMetricRequest.builder() + .metricType(MetricType.SPAN_TOKEN_USAGE) + .interval(TimeInterval.HOURLY) + .intervalStart(startTime) + .intervalEnd(endTime) + .breakdown(breakdown) + .build(), + Number.class, apiKey, workspaceName); + + var workspace = workspaceResourceClient.getWorkspaceSpanMetric( + spanRequest(MetricType.SPAN_TOKEN_USAGE, Set.of(projectId), startTime, endTime, breakdown), + apiKey, workspaceName); + + assertThat(workspace.results()).isNotEmpty(); + assertSeriesMatch(seriesFromWorkspace(workspace.results()), seriesFromProject(perProject.results())); + } + + @Test + void allProjects_matchSelectedProjects_whenWorkspaceContainsOnlyThose() { + var workspaceName = UUID.randomUUID().toString(); + var apiKey = UUID.randomUUID().toString(); + mockTargetWorkspace(apiKey, workspaceName, UUID.randomUUID().toString()); + + String projectName1 = RandomStringUtils.randomAlphabetic(10); + String projectName2 = RandomStringUtils.randomAlphabetic(10); + var projectId1 = projectResourceClient.createProject(projectName1, apiKey, workspaceName); + var projectId2 = projectResourceClient.createProject(projectName2, apiKey, workspaceName); + + createSpansWithUsage(projectName1, apiKey, workspaceName, Instant.now(), "completion_tokens", + List.of("openai", "anthropic")); + createSpansWithUsage(projectName2, apiKey, workspaceName, Instant.now(), "completion_tokens", + List.of("openai")); + + var endTime = Instant.now(); + var startTime = endTime.minus(Duration.ofDays(1)); + + var all = workspaceResourceClient.getWorkspaceSpanMetric( + spanRequest(MetricType.SPAN_TOKEN_USAGE, null, startTime, endTime, null), apiKey, workspaceName); + var selected = workspaceResourceClient.getWorkspaceSpanMetric( + spanRequest(MetricType.SPAN_TOKEN_USAGE, Set.of(projectId1, projectId2), startTime, endTime, null), + apiKey, workspaceName); + + assertThat(sumValues(all.results())).isPositive(); + assertSeriesMatch(seriesFromWorkspace(all.results()), seriesFromWorkspace(selected.results())); + } + + @Test + void workspaceMetric_isolatesByWorkspace() { + var workspaceA = UUID.randomUUID().toString(); + var apiKeyA = UUID.randomUUID().toString(); + mockTargetWorkspace(apiKeyA, workspaceA, UUID.randomUUID().toString()); + var workspaceB = UUID.randomUUID().toString(); + var apiKeyB = UUID.randomUUID().toString(); + mockTargetWorkspace(apiKeyB, workspaceB, UUID.randomUUID().toString()); + + String projectName = RandomStringUtils.randomAlphabetic(10); + projectResourceClient.createProject(projectName, apiKeyA, workspaceA); + createSpansWithUsage(projectName, apiKeyA, workspaceA, Instant.now(), "completion_tokens", + List.of("openai", "anthropic")); + + var endTime = Instant.now(); + var startTime = endTime.minus(Duration.ofDays(1)); + var request = spanRequest(MetricType.SPAN_TOKEN_USAGE, null, startTime, endTime, null); + + assertThat( + sumValues(workspaceResourceClient.getWorkspaceSpanMetric(request, apiKeyA, workspaceA).results())) + .isPositive(); + assertThat( + sumValues(workspaceResourceClient.getWorkspaceSpanMetric(request, apiKeyB, workspaceB).results())) + .isZero(); + } + + @Test + void spanTokenUsage_withFilter_matchesPerProjectEndpoint() { + var workspaceName = UUID.randomUUID().toString(); + var apiKey = UUID.randomUUID().toString(); + mockTargetWorkspace(apiKey, workspaceName, UUID.randomUUID().toString()); + + String projectName = RandomStringUtils.randomAlphabetic(10); + var projectId = projectResourceClient.createProject(projectName, apiKey, workspaceName); + + // Two openai spans and one anthropic span, each with 10 completion tokens, so a provider filter must + // narrow the total to the two openai spans => 20 tokens. + createSpansWithUsage(projectName, apiKey, workspaceName, Instant.now(), "completion_tokens", + List.of("openai", "anthropic", "openai")); + + var endTime = Instant.now(); + var startTime = endTime.minus(Duration.ofDays(1)); + List filters = List.of(SpanFilter.builder() + .field(SpanField.PROVIDER) + .operator(Operator.EQUAL) + .value("openai") + .build()); + + var perProject = projectMetricsResourceClient.getProjectMetrics(projectId, + ProjectMetricRequest.builder() + .metricType(MetricType.SPAN_TOKEN_USAGE) + .interval(TimeInterval.HOURLY) + .intervalStart(startTime) + .intervalEnd(endTime) + .spanFilters(filters) + .build(), + Number.class, apiKey, workspaceName); + + var workspace = workspaceResourceClient.getWorkspaceSpanMetric( + WorkspaceSpanMetricRequest.builder() + .metricType(MetricType.SPAN_TOKEN_USAGE) + .interval(TimeInterval.HOURLY) + .intervalStart(startTime) + .intervalEnd(endTime) + .projectIds(Set.of(projectId)) + .filters(filters) + .build(), + apiKey, workspaceName); + + assertThat(sumValues(workspace.results())).isEqualTo(20d); + assertSeriesMatch(seriesFromWorkspace(workspace.results()), seriesFromProject(perProject.results())); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" "}) + void tokenUsageBreakdown_withoutSubMetric_returnsBadRequest(String subMetric) { + var startTime = Instant.now().minus(Duration.ofDays(1)); + var breakdown = BreakdownConfig.builder().field(BreakdownField.PROVIDER).subMetric(subMetric).build(); + var request = spanRequest(MetricType.SPAN_TOKEN_USAGE, null, startTime, Instant.now(), breakdown); + + try (var response = workspaceResourceClient.callGetWorkspaceSpanMetric(request, API_KEY, WORKSPACE_NAME)) { + assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_BAD_REQUEST); + } + } + + // Only SPAN_TOKEN_USAGE is supported across projects; everything else (including other span metrics) is 400. + @ParameterizedTest + @EnumSource(value = MetricType.class, names = {"SPAN_COUNT", "SPAN_COST", "SPAN_DURATION", "TRACE_COUNT"}) + void unsupportedMetricType_returnsBadRequest(MetricType metricType) { + var startTime = Instant.now().minus(Duration.ofDays(1)); + var request = spanRequest(metricType, null, startTime, Instant.now(), null); + + try (var response = workspaceResourceClient.callGetWorkspaceSpanMetric(request, API_KEY, WORKSPACE_NAME)) { + assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_BAD_REQUEST); + } + } + + @Test + void missingMetricType_returnsBadRequest() { + var startTime = Instant.now().minus(Duration.ofDays(1)); + var request = spanRequest(null, null, startTime, Instant.now(), null); + + try (var response = workspaceResourceClient.callGetWorkspaceSpanMetric(request, API_KEY, WORKSPACE_NAME)) { + assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_BAD_REQUEST); + } + } + } + + private WorkspaceSpanMetricRequest spanRequest(MetricType metricType, Set projectIds, Instant startTime, + Instant endTime, BreakdownConfig breakdown) { + return WorkspaceSpanMetricRequest.builder() + .metricType(metricType) + .interval(TimeInterval.HOURLY) + .intervalStart(startTime) + .intervalEnd(endTime) + .projectIds(projectIds) + .breakdown(breakdown) + .build(); + } + + // Compares two sets of series as name -> (bucket -> value) maps, so the WorkspaceMetricResponse shape can be + // checked against the per-project ProjectMetricResponse (parity) or against another workspace response. + private void assertSeriesMatch(Map> actual, + Map> expected) { + assertThat(actual.keySet()).containsExactlyInAnyOrderElementsOf(expected.keySet()); + actual.forEach((name, points) -> { + var expectedPoints = expected.get(name); + assertThat(points.keySet()).containsExactlyInAnyOrderElementsOf(expectedPoints.keySet()); + points.forEach((bucket, value) -> assertThat(value == null ? 0d : value) + .isCloseTo(expectedPoints.get(bucket) == null ? 0d : expectedPoints.get(bucket), within(1e-6))); + }); + } + + private Map> seriesFromWorkspace(List results) { + return results.stream() + .collect(Collectors.toMap(WorkspaceMetricResponse.Result::name, result -> pointsToMap(result.data()))); + } + + private Map> seriesFromProject(List> results) { + return results.stream() + .collect(Collectors.toMap(ProjectMetricResponse.Results::name, result -> pointsToMap(result.data()))); + } + + private Map pointsToMap(List> data) { + return data.stream().collect( + HashMap::new, + (map, point) -> map.put(point.time().toString(), + point.value() == null ? null : point.value().doubleValue()), + HashMap::putAll); + } + + private double sumValues(List results) { + return results.stream() + .flatMap(result -> result.data().stream()) + .map(DataPoint::value) + .filter(value -> value != null) + .mapToDouble(Number::doubleValue) + .sum(); + } + + private List createSpansWithUsage(String projectName, String apiKey, String workspaceName, Instant time, + String usageKey, List providers) { + var spans = providers.stream() + .map(provider -> factory.manufacturePojo(Span.class).toBuilder() + .startTime(time) + .id(idGenerator.getTimeOrderedEpoch(time.toEpochMilli())) + .projectId(null) + .projectName(projectName) + .provider(provider) + .usage(Map.of(usageKey, 10)) + .feedbackScores(null) + .build()) + .toList(); + + spanResourceClient.batchCreateSpans(spans, apiKey, workspaceName); + + return spans; + } + private List createSpans(String projectName, String apiKey, String workspaceName, Instant time) { var spans = PodamFactoryUtils.manufacturePojoList(factory, Span.class) diff --git a/apps/opik-frontend/src/api/projects/useMetricData.ts b/apps/opik-frontend/src/api/projects/useMetricData.ts new file mode 100644 index 00000000000..78e08143462 --- /dev/null +++ b/apps/opik-frontend/src/api/projects/useMetricData.ts @@ -0,0 +1,74 @@ +import useProjectMetric, { + INTERVAL_TYPE, + METRIC_NAME_TYPE, + ProjectMetricsResponse, +} from "@/api/projects/useProjectMetric"; +import useWorkspaceMetric from "@/api/projects/useWorkspaceMetric"; +import { Filter } from "@/types/filters"; +import { LOGS_SOURCE } from "@/types/traces"; +import { BreakdownConfig } from "@/types/dashboard"; +import { QueryConfig } from "@/api/api"; + +type UseMetricDataParams = { + // Exactly one of projectId / projectIds identifies the scope. + projectId?: string; + projectIds?: string[]; + metricName: METRIC_NAME_TYPE; + interval: INTERVAL_TYPE; + intervalStart: string | undefined; + intervalEnd: string | undefined; + traceFilters?: Filter[]; + threadFilters?: Filter[]; + spanFilters?: Filter[]; + breakdown?: BreakdownConfig; + logsSource?: LOGS_SOURCE; +}; + +// Picks the right metrics source for a dashboard metric widget: a single project uses the per-project endpoint (all +// metrics, filters, log source); a project set uses the workspace endpoint (span metrics aggregated across projects). +// Both return the same ProjectMetricsResponse shape, so callers consume one result regardless of scope. +const useMetricData = ( + params: UseMetricDataParams, + config?: QueryConfig, +) => { + const isWorkspace = Array.isArray(params.projectIds); + + const projectQuery = useProjectMetric( + { + projectId: params.projectId ?? "", + metricName: params.metricName, + interval: params.interval, + intervalStart: params.intervalStart, + intervalEnd: params.intervalEnd, + traceFilters: params.traceFilters, + threadFilters: params.threadFilters, + spanFilters: params.spanFilters, + breakdown: params.breakdown, + logsSource: params.logsSource, + }, + { + ...config, + enabled: !isWorkspace && !!params.projectId && (config?.enabled ?? true), + }, + ); + + const workspaceQuery = useWorkspaceMetric( + { + projectIds: params.projectIds ?? [], + metricName: params.metricName, + interval: params.interval, + intervalStart: params.intervalStart, + intervalEnd: params.intervalEnd, + breakdown: params.breakdown, + spanFilters: params.spanFilters, + }, + { + ...config, + enabled: isWorkspace && (config?.enabled ?? true), + }, + ); + + return isWorkspace ? workspaceQuery : projectQuery; +}; + +export default useMetricData; diff --git a/apps/opik-frontend/src/api/projects/useWorkspaceMetric.ts b/apps/opik-frontend/src/api/projects/useWorkspaceMetric.ts new file mode 100644 index 00000000000..f0f49bcadae --- /dev/null +++ b/apps/opik-frontend/src/api/projects/useWorkspaceMetric.ts @@ -0,0 +1,78 @@ +import { QueryFunctionContext, useQuery } from "@tanstack/react-query"; +import api, { WORKSPACES_REST_ENDPOINT, QueryConfig } from "@/api/api"; +import { BreakdownConfig, BREAKDOWN_FIELD } from "@/types/dashboard"; +import { Filter } from "@/types/filters"; +import { processFiltersArray } from "@/lib/filters"; +import { + INTERVAL_TYPE, + METRIC_NAME_TYPE, + ProjectMetricsResponse, +} from "@/api/projects/useProjectMetric"; + +type UseWorkspaceMetricParams = { + projectIds: string[]; + metricName: METRIC_NAME_TYPE; + interval: INTERVAL_TYPE; + intervalStart: string | undefined; + intervalEnd: string | undefined; + breakdown?: BreakdownConfig; + spanFilters?: Filter[]; +}; + +const processBreakdownConfig = (breakdown?: BreakdownConfig) => { + if (!breakdown || breakdown.field === BREAKDOWN_FIELD.NONE) { + return undefined; + } + + return { + field: breakdown.field, + ...(breakdown.metadataKey && { metadata_key: breakdown.metadataKey }), + ...(breakdown.subMetric && { sub_metric: breakdown.subMetric }), + }; +}; + +const getWorkspaceMetric = async ( + { signal }: QueryFunctionContext, + { + projectIds, + metricName, + interval, + intervalStart, + intervalEnd, + breakdown, + spanFilters, + }: UseWorkspaceMetricParams, +) => { + const processedFilters = processFiltersArray(spanFilters ?? []); + const { data } = await api.post( + `${WORKSPACES_REST_ENDPOINT}metrics/spans`, + { + metric_type: metricName, + interval, + ...(intervalStart && { interval_start: intervalStart }), + ...(intervalEnd && { interval_end: intervalEnd }), + // Empty => all projects in the workspace; otherwise only the selected set + ...(projectIds.length > 0 && { project_ids: projectIds }), + breakdown: processBreakdownConfig(breakdown), + ...(processedFilters.length > 0 && { filters: processedFilters }), + }, + { + signal, + }, + ); + + return data; +}; + +const useWorkspaceMetric = ( + params: UseWorkspaceMetricParams, + config?: QueryConfig, +) => { + return useQuery({ + queryKey: ["workspaceMetrics", params], + queryFn: (context) => getWorkspaceMetric(context, params), + ...config, + }); +}; + +export default useWorkspaceMetric; diff --git a/apps/opik-frontend/src/lib/dashboard/workspaceMetrics.ts b/apps/opik-frontend/src/lib/dashboard/workspaceMetrics.ts new file mode 100644 index 00000000000..13c78ebb9f0 --- /dev/null +++ b/apps/opik-frontend/src/lib/dashboard/workspaceMetrics.ts @@ -0,0 +1,90 @@ +import { METRIC_NAME_TYPE } from "@/api/projects/useProjectMetric"; + +// Metric supported when aggregating across projects via the workspace endpoint +// (POST /v1/private/workspaces/metrics/spans). Scoped to span token usage, with provider/model breakdown and filters. +export const WORKSPACE_METRIC_NAMES: METRIC_NAME_TYPE[] = [ + METRIC_NAME_TYPE.SPAN_TOKEN_USAGE, +]; + +export const isWorkspaceMetric = (metric?: string): boolean => + WORKSPACE_METRIC_NAMES.includes(metric as METRIC_NAME_TYPE); + +// Metric options for the time-series widget editor when multiple projects are selected. +export const WORKSPACE_TIME_SERIES_METRIC_OPTIONS = [ + { + value: METRIC_NAME_TYPE.SPAN_TOKEN_USAGE, + label: "Span token usage", + filterType: "span" as const, + }, +]; + +// Metric definitions for the single-metric (stat card) widget when multiple projects are selected. These render a +// single total from the workspace endpoint queried with interval=TOTAL, so they carry their own value formatters. +export type WorkspaceStatMetricDefinition = { + value: METRIC_NAME_TYPE; + label: string; + formatter: (value: number) => string; + tooltipFormatter?: (value: number) => string; + // Token usage returns one value per usage key, so the card needs a selected key (e.g. total_tokens). + requiresUsageKey?: boolean; +}; + +export const WORKSPACE_STAT_METRICS: WorkspaceStatMetricDefinition[] = [ + { + value: METRIC_NAME_TYPE.SPAN_TOKEN_USAGE, + label: "Total span token usage", + formatter: (value: number) => value.toLocaleString(), + requiresUsageKey: true, + }, +]; + +export const WORKSPACE_STAT_METRIC_OPTIONS = WORKSPACE_STAT_METRICS.map( + (m) => ({ + value: m.value, + label: m.label, + }), +); + +export const getWorkspaceStatMetric = ( + metricValue?: string, +): WorkspaceStatMetricDefinition | null => + WORKSPACE_STAT_METRICS.find((m) => m.value === metricValue) || null; + +export const DEFAULT_WORKSPACE_USAGE_METRIC = "total_tokens"; + +export type ResolvedProjectSelection = { + // Single-project mode (per-project endpoints). + projectId?: string; + // Workspace mode (aggregate across projects). Present only when 2+ projects are selected. + projectIds?: string[]; +}; + +// Resolves a widget's project configuration into either a single project or a project set. A runtime project (e.g. +// the project Insights tab) always pins to a single project. Otherwise: one selected project => single; two or more +// => workspace aggregation; none => not configured. Falls back to the legacy single `projectId` field. +export const resolveProjectSelection = ({ + runtimeProjectId, + projectId, + projectIds, +}: { + runtimeProjectId?: string; + projectId?: string; + projectIds?: string[]; +}): ResolvedProjectSelection => { + if (runtimeProjectId) { + return { projectId: runtimeProjectId }; + } + if (Array.isArray(projectIds)) { + if (projectIds.length === 1) return { projectId: projectIds[0] }; + if (projectIds.length >= 2) return { projectIds }; + return {}; + } + if (projectId) return { projectId }; + return {}; +}; + +// Whether the current editor selection aggregates across projects (2+ projects and not pinned to a runtime project). +export const isMultiProjectSelection = ( + runtimeProjectId: string | undefined, + projectIds: string[], +): boolean => !runtimeProjectId && projectIds.length >= 2; diff --git a/apps/opik-frontend/src/types/dashboard.ts b/apps/opik-frontend/src/types/dashboard.ts index 8c72628d1ea..d6f217fb689 100644 --- a/apps/opik-frontend/src/types/dashboard.ts +++ b/apps/opik-frontend/src/types/dashboard.ts @@ -73,7 +73,11 @@ export interface BreakdownConfig { export interface ProjectMetricsWidget { type: WIDGET_TYPE.PROJECT_METRICS; config: { + // Single-project selection (per-project metrics endpoint). projectId?: string; + // Workspace-level selection (aggregate across projects). When present, the widget queries the + // workspace metrics endpoint; an empty array means "all projects in the workspace". + projectIds?: string[]; metricType: string; chartType?: CHART_TYPE.line | CHART_TYPE.bar; traceFilters?: Filters; @@ -97,8 +101,12 @@ export interface ProjectStatsCardWidget { type: WIDGET_TYPE.PROJECT_STATS_CARD; config: { source: TRACE_DATA_TYPE; - projectId: string; + // Single-project selection (stats endpoint). Workspace/multi-project selection uses projectIds. + projectId?: string; + projectIds?: string[]; metric: string; + // Selected token-usage key when metric is the workspace span-token-usage total. + usageMetric?: string; traceFilters?: Filters; spanFilters?: Filters; } & Record; diff --git a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/MetricChart/MetricChartContainer.tsx b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/MetricChart/MetricChartContainer.tsx index 1926bdacc5e..b545c3b3574 100644 --- a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/MetricChart/MetricChartContainer.tsx +++ b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/MetricChart/MetricChartContainer.tsx @@ -12,10 +12,11 @@ import { formatNumericData } from "@/lib/utils"; import { TransformedData } from "@/types/projects"; import useChartConfig from "@/hooks/useChartConfig"; -import useProjectMetric, { +import { INTERVAL_TYPE, METRIC_NAME_TYPE, } from "@/api/projects/useProjectMetric"; +import useMetricData from "@/api/projects/useMetricData"; import { ChartTooltipRenderValueArguments } from "@/shared/Charts/ChartTooltipContent/ChartTooltipContent"; import NoData from "@/shared/NoData/NoData"; import { ValueType } from "recharts/types/component/DefaultTooltipContent"; @@ -39,7 +40,10 @@ interface MetricContainerChartProps { name: string; description: string; chartType: CHART_TYPE.line | CHART_TYPE.bar; - projectId: string; + // Single-project mode (per-project endpoint). Mutually exclusive with projectIds. + projectId?: string; + // Workspace mode (aggregate across projects). Empty array => all projects in the workspace. + projectIds?: string[]; interval: INTERVAL_TYPE; intervalStart: string | undefined; intervalEnd: string | undefined; @@ -81,6 +85,7 @@ const MetricContainerChart = ({ description, metricName, projectId, + projectIds, interval, intervalStart, intervalEnd, @@ -99,9 +104,10 @@ const MetricContainerChart = ({ isAggregateTotal = false, customEmptyState, }: MetricContainerChartProps) => { - const { data: response, isPending } = useProjectMetric( + const { data: response, isPending } = useMetricData( { projectId, + projectIds, metricName, interval, intervalStart, @@ -111,10 +117,7 @@ const MetricContainerChart = ({ spanFilters, breakdown, }, - { - enabled: !!projectId, - refetchInterval: 30000, - }, + { refetchInterval: 30000 }, ); const traces = response?.results; diff --git a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx index d11dbd0fca4..d1a7cfaa0e6 100644 --- a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx +++ b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx @@ -34,6 +34,11 @@ import { import get from "lodash/get"; import { METRIC_NAME_TYPE } from "@/api/projects/useProjectMetric"; +import { + WORKSPACE_TIME_SERIES_METRIC_OPTIONS, + isWorkspaceMetric, + isMultiProjectSelection, +} from "@/lib/dashboard/workspaceMetrics"; import useProjectTokenUsageNames from "@/api/projects/useProjectTokenUsageNames"; import { DEFAULT_DATE_PRESET } from "@/v1/pages-shared/traces/MetricDateRangeSelect/constants"; import { @@ -134,6 +139,12 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { const metricType = config.metricType || ""; const chartType = config.chartType || CHART_TYPE.line; const localProjectId = config.projectId; + const localProjectIds = useMemo( + () => + (config.projectIds as string[] | undefined) ?? + (localProjectId ? [localProjectId] : []), + [config.projectIds, localProjectId], + ); const traceFilters = useMemo( () => (config.traceFilters as Filter[] | undefined) || [], @@ -177,7 +188,18 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { }; }); const hasRuntimeProjectId = !!runtimeContext.projectId; - const projectId = runtimeContext.projectId || localProjectId || ""; + // Representative project for loading option lists (usage keys, feedback scores, filter/breakdown autocompletes) + // and the preview: the runtime project, else the first selected project. + const projectId = runtimeContext.projectId || localProjectIds[0] || ""; + + // Selecting more than one project aggregates across projects, which only supports span metrics. + const isMultiProject = isMultiProjectSelection( + runtimeContext.projectId, + localProjectIds, + ); + const metricOptions = isMultiProject + ? WORKSPACE_TIME_SERIES_METRIC_OPTIONS + : METRIC_OPTIONS; const selectedMetric = METRIC_OPTIONS.find((m) => m.value === metricType); const isTraceMetric = !metricType || selectedMetric?.filterType === "trace"; @@ -240,7 +262,7 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { defaultValues: { metricType, chartType, - projectId, + projectIds: localProjectIds, traceFilters, threadFilters, spanFilters, @@ -327,13 +349,38 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { }); }; - const handleProjectChange = (projectId: string) => { - updatePreviewWidget({ - config: { - ...config, - projectId, - }, - }); + const handleProjectsChange = (projectIds: string[]) => { + // projectIds is the single source of truth for selection; drop the legacy single-project field. + const nextConfig = { ...config, projectIds }; + delete nextConfig.projectId; + form.setValue("projectIds", projectIds); + + // Aggregating across projects only supports span metrics. If the selection becomes multi-project while a + // non-span metric is chosen, fall back to span token usage and clear metric-specific selections. + const becomesMultiProject = isMultiProjectSelection( + runtimeContext.projectId, + projectIds, + ); + if (becomesMultiProject && !isWorkspaceMetric(metricType)) { + nextConfig.metricType = METRIC_NAME_TYPE.SPAN_TOKEN_USAGE; + nextConfig.traceFilters = []; + nextConfig.threadFilters = []; + nextConfig.spanFilters = []; + nextConfig.feedbackScores = []; + nextConfig.durationMetrics = []; + nextConfig.usageMetrics = []; + nextConfig.breakdown = { field: BREAKDOWN_FIELD.NONE }; + form.setValue("metricType", METRIC_NAME_TYPE.SPAN_TOKEN_USAGE); + form.setValue("breakdown.field", BREAKDOWN_FIELD.NONE); + form.setValue("traceFilters", []); + form.setValue("threadFilters", []); + form.setValue("spanFilters", []); + form.setValue("feedbackScores", []); + form.setValue("durationMetrics", []); + form.setValue("usageMetrics", []); + } + + updatePreviewWidget({ config: nextConfig }); }; const handleFeedbackScoresChange = (newFeedbackScores: string[]) => { @@ -408,9 +455,9 @@ const ProjectMetricsEditor = forwardRef((_, ref) => {
{ - const validationErrors = get(formState.errors, ["projectId"]); + const validationErrors = get(formState.errors, ["projectIds"]); return ( Project @@ -419,10 +466,13 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { className={cn("flex-1", { "border-destructive": Boolean(validationErrors?.message), })} - value={field.value || ""} + multiselect + showSelectAll + selectAllLabel="All projects" + value={field.value || []} onValueChange={(value) => { field.onChange(value); - handleProjectChange(value); + handleProjectsChange(value); }} disabled={hasRuntimeProjectId} /> @@ -451,7 +501,7 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { field.onChange(value); handleMetricTypeChange(value); }} - options={METRIC_OPTIONS} + options={metricOptions} placeholder="Select a metric type" /> diff --git a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx index 772bbe1ed1e..91d3f90a57b 100644 --- a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx +++ b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx @@ -7,6 +7,10 @@ import { INTERVAL_TYPE, METRIC_NAME_TYPE, } from "@/api/projects/useProjectMetric"; +import { + isWorkspaceMetric, + resolveProjectSelection, +} from "@/lib/dashboard/workspaceMetrics"; import { useDashboardStore, selectRuntimeConfig, @@ -80,22 +84,25 @@ const ProjectMetricsWidget: React.FunctionComponent< }, [sectionId, widgetId, onAddEditWidgetCallback]); const widgetProjectId = widget?.config?.projectId as string | undefined; + const widgetProjectIds = widget?.config?.projectIds as string[] | undefined; + + const { projectId, projectIds } = useMemo( + () => + resolveProjectSelection({ + runtimeProjectId: runtimeContext.projectId, + projectId: widgetProjectId, + projectIds: widgetProjectIds, + }), + [widgetProjectId, widgetProjectIds, runtimeContext.projectId], + ); - const { projectId, interval, intervalStart, intervalEnd } = useMemo(() => { - const resolvedProjectId = - runtimeContext.projectId || widgetProjectId || undefined; - - const { interval, intervalStart, intervalEnd } = calculateIntervalConfig( - runtimeContext.dateRange, - ); + const { interval, intervalStart, intervalEnd } = useMemo( + () => calculateIntervalConfig(runtimeContext.dateRange), + [runtimeContext.dateRange], + ); - return { - projectId: resolvedProjectId, - interval, - intervalStart, - intervalEnd, - }; - }, [widgetProjectId, runtimeContext.projectId, runtimeContext.dateRange]); + const isWorkspaceMode = Array.isArray(projectIds); + const isConfigured = Boolean(projectId) || isWorkspaceMode; const metricType = widget?.config?.metricType as string | undefined; const metricName = metricType as METRIC_NAME_TYPE | undefined; @@ -396,7 +403,7 @@ const ProjectMetricsWidget: React.FunctionComponent< const threadFilters = widget.config?.threadFilters as Filter[] | undefined; const spanFilters = widget.config?.spanFilters as Filter[] | undefined; - if (!projectId) { + if (!isConfigured) { return ( + ); + } + const validTraceFilters = traceFilters?.filter(isFilterValid); const validThreadFilters = threadFilters?.filter(isFilterValid); const validSpanFilters = spanFilters?.filter(isFilterValid); @@ -444,7 +462,8 @@ const ProjectMetricsWidget: React.FunctionComponent< isAggregateTotal={!!isAggregateTotal} intervalStart={intervalStart} intervalEnd={intervalEnd} - projectId={projectId!} + projectId={projectId} + projectIds={projectIds} chartType={chartType} traceFilters={validTraceFilters} threadFilters={validThreadFilters} diff --git a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts index ecc7433fa3a..c3bc0cc18d6 100644 --- a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts +++ b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts @@ -31,6 +31,7 @@ export const ProjectMetricsWidgetSchema = z.object({ .min(1, { message: "Metric type is required" }), chartType: z.nativeEnum(CHART_TYPE), projectId: z.string().optional(), + projectIds: z.array(z.string()).optional(), traceFilters: FiltersArraySchema.optional(), threadFilters: FiltersArraySchema.optional(), spanFilters: FiltersArraySchema.optional(), diff --git a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx index f4ae6d90cf2..7deef8eea45 100644 --- a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx +++ b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx @@ -41,7 +41,15 @@ import { ProjectStatsCardWidgetFormData, } from "./schema"; import { getAllMetricOptions } from "./metrics"; +import { + getWorkspaceStatMetric, + WORKSPACE_STAT_METRIC_OPTIONS, + isMultiProjectSelection, + DEFAULT_WORKSPACE_USAGE_METRIC, +} from "@/lib/dashboard/workspaceMetrics"; import useTracesOrSpansScoresColumns from "@/hooks/useTracesOrSpansScoresColumns"; +import useProjectTokenUsageNames from "@/api/projects/useProjectTokenUsageNames"; +import { METRIC_NAME_TYPE } from "@/api/projects/useProjectMetric"; import { TRACE_DATA_TYPE } from "@/constants/traces"; const ProjectStatsCardEditor = forwardRef((_, ref) => { @@ -53,7 +61,14 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { const { config } = widgetData; const source = config.source || TRACE_DATA_TYPE.traces; const metric = config.metric || ""; + const usageMetric = config.usageMetric || ""; const localProjectId = config.projectId; + const localProjectIds = useMemo( + () => + (config.projectIds as string[] | undefined) ?? + (localProjectId ? [localProjectId] : []), + [config.projectIds, localProjectId], + ); const traceFilters = useMemo( () => config.traceFilters || [], @@ -71,21 +86,51 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }; }); const hasRuntimeProjectId = !!runtimeContext.projectId; - const projectId = runtimeContext.projectId || localProjectId || ""; + // Representative project (runtime, else first selected) for loading option lists. + const projectId = runtimeContext.projectId || localProjectIds[0] || ""; const isTraceSource = source === TRACE_DATA_TYPE.traces; + // Selecting more than one project aggregates across projects, which only supports span total metrics. + const isMultiProject = isMultiProjectSelection( + runtimeContext.projectId, + localProjectIds, + ); + const workspaceMetricDef = isMultiProject + ? getWorkspaceStatMetric(metric) + : null; + const requiresUsageKey = Boolean(workspaceMetricDef?.requiresUsageKey); + // Multi-project aggregates span metrics, so filters are always span filters there; single-project follows source. + const useSpanFilters = isMultiProject || !isTraceSource; + const { data, isPending } = useTracesOrSpansScoresColumns( { projectId, type: source, }, - {}, + { enabled: !isMultiProject && !!projectId }, + ); + + const { data: tokenUsageNamesData } = useProjectTokenUsageNames( + { projectId }, + { enabled: isMultiProject && requiresUsageKey && !!projectId }, + ); + + const usageKeyOptions = useMemo( + () => + (tokenUsageNamesData?.names ?? []) + .slice() + .sort((a, b) => a.localeCompare(b)) + .map((name) => ({ value: name, label: name })), + [tokenUsageNamesData?.names], ); const metricOptions = useMemo(() => { + if (isMultiProject) { + return WORKSPACE_STAT_METRIC_OPTIONS; + } const scoreNames = data?.scores.map((s) => s.name) || []; return getAllMetricOptions(source, scoreNames); - }, [source, data?.scores]); + }, [isMultiProject, source, data?.scores]); const form = useForm({ resolver: zodResolver(ProjectStatsCardWidgetSchema), @@ -93,7 +138,8 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { defaultValues: { source, metric, - projectId, + projectIds: localProjectIds, + usageMetric, traceFilters, spanFilters, }, @@ -126,20 +172,50 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }); }; - const handleProjectChange = (projectId: string) => { - updatePreviewWidget({ - config: { - ...config, - projectId, - }, - }); + const handleProjectsChange = (projectIds: string[]) => { + const nextConfig = { ...config, projectIds }; + delete nextConfig.projectId; + form.setValue("projectIds", projectIds); + + const becomesMultiProject = !hasRuntimeProjectId && projectIds.length >= 2; + if (becomesMultiProject && !getWorkspaceStatMetric(metric)) { + // Switching into multi-project with an incompatible metric: default to span token usage total. + nextConfig.metric = METRIC_NAME_TYPE.SPAN_TOKEN_USAGE; + nextConfig.usageMetric = DEFAULT_WORKSPACE_USAGE_METRIC; + nextConfig.traceFilters = []; + nextConfig.spanFilters = []; + form.setValue("metric", METRIC_NAME_TYPE.SPAN_TOKEN_USAGE); + form.setValue("usageMetric", DEFAULT_WORKSPACE_USAGE_METRIC); + form.setValue("traceFilters", []); + form.setValue("spanFilters", []); + } else if (!becomesMultiProject && getWorkspaceStatMetric(metric)) { + // Leaving multi-project with a workspace-only metric selected: clear it so the user picks a valid one. + nextConfig.metric = ""; + form.setValue("metric", ""); + } + + updatePreviewWidget({ config: nextConfig }); }; const handleMetricChange = (value: string) => { + const nextConfig = { ...config, metric: value }; + // Picking the token-usage total needs a usage key; default it. + if ( + isMultiProject && + getWorkspaceStatMetric(value)?.requiresUsageKey && + !config.usageMetric + ) { + nextConfig.usageMetric = DEFAULT_WORKSPACE_USAGE_METRIC; + form.setValue("usageMetric", DEFAULT_WORKSPACE_USAGE_METRIC); + } + updatePreviewWidget({ config: nextConfig }); + }; + + const handleUsageMetricChange = (value: string) => { updatePreviewWidget({ config: { ...config, - metric: value, + usageMetric: value, }, }); }; @@ -149,9 +225,9 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => {
{ - const validationErrors = get(formState.errors, ["projectId"]); + const validationErrors = get(formState.errors, ["projectIds"]); return ( Project @@ -160,10 +236,13 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { className={cn("flex-1", { "border-destructive": Boolean(validationErrors?.message), })} - value={field.value || ""} + multiselect + showSelectAll + selectAllLabel="All projects" + value={field.value || []} onValueChange={(value) => { field.onChange(value); - handleProjectChange(value); + handleProjectsChange(value); }} disabled={hasRuntimeProjectId} /> @@ -174,28 +253,66 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }} /> + {!isMultiProject && ( + { + const validationErrors = get(formState.errors, ["source"]); + return ( + + Source + + { + field.onChange(value); + handleSourceChange(value); + }} + options={SOURCE_OPTIONS} + placeholder="Select source" + renderOption={renderSourceOption} + renderTrigger={renderSourceTrigger} + /> + + + + ); + }} + /> + )} + { - const validationErrors = get(formState.errors, ["source"]); + const validationErrors = get(formState.errors, ["metric"]); + const placeholder = + !isMultiProject && isPending + ? "Loading available metrics..." + : "Select a metric"; + return ( - Source + Metric type - { field.onChange(value); - handleSourceChange(value); + handleMetricChange(value); }} - options={SOURCE_OPTIONS} - placeholder="Select source" - renderOption={renderSourceOption} - renderTrigger={renderSourceTrigger} + options={metricOptions} + placeholder={placeholder} + disabled={!isMultiProject && isPending} /> @@ -204,51 +321,42 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }} /> - { - const validationErrors = get(formState.errors, ["metric"]); - const placeholder = isPending - ? "Loading available metrics..." - : "Select a metric"; - - return ( + {isMultiProject && requiresUsageKey && ( + ( - Metric type + Usage metric { field.onChange(value); - handleMetricChange(value); + handleUsageMetricChange(value); }} - options={metricOptions} - placeholder={placeholder} - disabled={isPending} + options={usageKeyOptions} + placeholder="Select a usage metric" /> - ); - }} - /> + )} + /> + )} { updatePreviewWidget({ config: { ...config, - ...(isTraceSource - ? { traceFilters: filters } - : { spanFilters: filters }), + ...(useSpanFilters + ? { spanFilters: filters } + : { traceFilters: filters }), }, }); }} diff --git a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx index 5af8f8d18d6..bcc013ba588 100644 --- a/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx +++ b/apps/opik-frontend/src/v1/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx @@ -15,6 +15,11 @@ import { calculateIntervalConfig } from "@/v1/pages-shared/traces/MetricDateRang import { DEFAULT_DATE_PRESET } from "@/v1/pages-shared/traces/MetricDateRangeSelect/constants"; import useTracesStatistic from "@/api/traces/useTracesStatistic"; import useSpansStatistic from "@/api/traces/useSpansStatistic"; +import useWorkspaceMetric from "@/api/projects/useWorkspaceMetric"; +import { + INTERVAL_TYPE, + METRIC_NAME_TYPE, +} from "@/api/projects/useProjectMetric"; import { ColumnStatistic } from "@/types/shared"; import { Skeleton } from "@/ui/skeleton"; import { TRACE_DATA_TYPE } from "@/constants/traces"; @@ -25,6 +30,11 @@ import { isFeedbackScoreMetric, extractFeedbackScoreName, } from "./metrics"; +import { + getWorkspaceStatMetric, + resolveProjectSelection, + DEFAULT_WORKSPACE_USAGE_METRIC, +} from "@/lib/dashboard/workspaceMetrics"; import { formatScoreDisplay } from "@/lib/feedback-scores"; const renderMetricDisplay = ( @@ -80,24 +90,32 @@ const ProjectStatsCardWidget: React.FunctionComponent< }, [sectionId, widgetId, onAddEditWidgetCallback]); const widgetProjectId = widget?.config?.projectId as string | undefined; + const widgetProjectIds = widget?.config?.projectIds as string[] | undefined; - const { projectId, intervalStart, intervalEnd } = useMemo(() => { - const resolvedProjectId = - runtimeContext.projectId || widgetProjectId || undefined; + const { projectId, projectIds } = useMemo( + () => + resolveProjectSelection({ + runtimeProjectId: runtimeContext.projectId, + projectId: widgetProjectId, + projectIds: widgetProjectIds, + }), + [widgetProjectId, widgetProjectIds, runtimeContext.projectId], + ); - const { intervalStart, intervalEnd } = calculateIntervalConfig( - runtimeContext.dateRange, - ); + const { intervalStart, intervalEnd } = useMemo( + () => calculateIntervalConfig(runtimeContext.dateRange), + [runtimeContext.dateRange], + ); - return { - projectId: resolvedProjectId, - intervalStart, - intervalEnd, - }; - }, [widgetProjectId, runtimeContext.projectId, runtimeContext.dateRange]); + const isMultiProject = Array.isArray(projectIds); + const isConfigured = Boolean(projectId) || isMultiProject; const source = widget?.config?.source as TRACE_DATA_TYPE | undefined; const metric = widget?.config?.metric as string | undefined; + const usageMetric = widget?.config?.usageMetric as string | undefined; + const workspaceMetricDef = isMultiProject + ? getWorkspaceStatMetric(metric ?? "") + : null; const traceFilters = widget?.config?.traceFilters as Filter[] | undefined; const spanFilters = widget?.config?.spanFilters as Filter[] | undefined; @@ -117,7 +135,10 @@ const ProjectStatsCardWidget: React.FunctionComponent< fromTime: intervalStart, toTime: intervalEnd, }, - { enabled: source === TRACE_DATA_TYPE.traces && !!projectId }, + { + enabled: + !isMultiProject && source === TRACE_DATA_TYPE.traces && !!projectId, + }, ); const spansStatistic = useSpansStatistic( @@ -127,7 +148,22 @@ const ProjectStatsCardWidget: React.FunctionComponent< fromTime: intervalStart, toTime: intervalEnd, }, - { enabled: source === TRACE_DATA_TYPE.spans && !!projectId }, + { + enabled: + !isMultiProject && source === TRACE_DATA_TYPE.spans && !!projectId, + }, + ); + + const workspaceMetric = useWorkspaceMetric( + { + projectIds: projectIds ?? [], + metricName: (metric as METRIC_NAME_TYPE) ?? METRIC_NAME_TYPE.SPAN_COUNT, + interval: INTERVAL_TYPE.TOTAL, + intervalStart, + intervalEnd, + spanFilters: validSpanFilters, + }, + { enabled: isMultiProject && Boolean(workspaceMetricDef) }, ); const { data, isLoading, error } = useMemo( @@ -148,8 +184,50 @@ const ProjectStatsCardWidget: React.FunctionComponent< /> ) : undefined; + const renderMultiProjectContent = () => { + if (!metric || !workspaceMetricDef) { + return ( + + ); + } + + if (workspaceMetric.isLoading) { + return ( +
+
+ + +
+
+ ); + } + + const results = workspaceMetric.data?.results ?? []; + // Token usage returns one series per usage key; pick the selected key. Count/cost return a single series. + const series = workspaceMetricDef.requiresUsageKey + ? results.find( + (r) => r.name === (usageMetric || DEFAULT_WORKSPACE_USAGE_METRIC), + ) + : results[0]; + const value = series?.data?.[0]?.value; + + return renderMetricDisplay( + workspaceMetricDef.label, + value !== undefined && value !== null + ? workspaceMetricDef.formatter(Number(value)) + : "-", + value !== undefined && value !== null + ? workspaceMetricDef.tooltipFormatter?.(Number(value)) + : undefined, + ); + }; + const renderCardContent = () => { - if (!projectId) { + if (!isConfigured) { return ( all projects in the workspace. + projectIds?: string[]; interval: INTERVAL_TYPE; intervalStart: string | undefined; intervalEnd: string | undefined; @@ -92,6 +96,7 @@ const MetricContainerChart = ({ description, metricName, projectId, + projectIds, interval, intervalStart, intervalEnd, @@ -120,9 +125,10 @@ const MetricContainerChart = ({ hideXAxis, hideYAxis, }: MetricContainerChartProps) => { - const { data: response, isPending } = useProjectMetric( + const { data: response, isPending } = useMetricData( { projectId, + projectIds, metricName, interval, intervalStart, @@ -133,10 +139,7 @@ const MetricContainerChart = ({ breakdown, logsSource, }, - { - enabled: !!projectId, - refetchInterval: 30000, - }, + { refetchInterval: 30000 }, ); const traces = response?.results; diff --git a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx index 74d5c1d91ed..0fe18d162ea 100644 --- a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx +++ b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsEditor.tsx @@ -34,6 +34,11 @@ import { import get from "lodash/get"; import { METRIC_NAME_TYPE } from "@/api/projects/useProjectMetric"; +import { + WORKSPACE_TIME_SERIES_METRIC_OPTIONS, + isWorkspaceMetric, + isMultiProjectSelection, +} from "@/lib/dashboard/workspaceMetrics"; import useProjectTokenUsageNames from "@/api/projects/useProjectTokenUsageNames"; import { DEFAULT_DATE_PRESET } from "@/v2/pages-shared/traces/MetricDateRangeSelect/constants"; import { @@ -134,6 +139,12 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { const metricType = config.metricType || ""; const chartType = config.chartType || CHART_TYPE.line; const localProjectId = config.projectId; + const localProjectIds = useMemo( + () => + (config.projectIds as string[] | undefined) ?? + (localProjectId ? [localProjectId] : []), + [config.projectIds, localProjectId], + ); const traceFilters = useMemo( () => (config.traceFilters as Filter[] | undefined) || [], @@ -177,7 +188,18 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { }; }); const hasRuntimeProjectId = !!runtimeContext.projectId; - const projectId = runtimeContext.projectId || localProjectId || ""; + // Representative project for loading option lists (usage keys, feedback scores, filter/breakdown autocompletes) + // and the preview: the runtime project, else the first selected project. + const projectId = runtimeContext.projectId || localProjectIds[0] || ""; + + // Selecting more than one project aggregates across projects, which only supports span metrics. + const isMultiProject = isMultiProjectSelection( + runtimeContext.projectId, + localProjectIds, + ); + const metricOptions = isMultiProject + ? WORKSPACE_TIME_SERIES_METRIC_OPTIONS + : METRIC_OPTIONS; const selectedMetric = METRIC_OPTIONS.find((m) => m.value === metricType); const isTraceMetric = !metricType || selectedMetric?.filterType === "trace"; @@ -240,7 +262,7 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { defaultValues: { metricType, chartType, - projectId, + projectIds: localProjectIds, traceFilters, threadFilters, spanFilters, @@ -327,13 +349,38 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { }); }; - const handleProjectChange = (projectId: string) => { - updatePreviewWidget({ - config: { - ...config, - projectId, - }, - }); + const handleProjectsChange = (projectIds: string[]) => { + // projectIds is the single source of truth for selection; drop the legacy single-project field. + const nextConfig = { ...config, projectIds }; + delete nextConfig.projectId; + form.setValue("projectIds", projectIds); + + // Aggregating across projects only supports span metrics. If the selection becomes multi-project while a + // non-span metric is chosen, fall back to span token usage and clear metric-specific selections. + const becomesMultiProject = isMultiProjectSelection( + runtimeContext.projectId, + projectIds, + ); + if (becomesMultiProject && !isWorkspaceMetric(metricType)) { + nextConfig.metricType = METRIC_NAME_TYPE.SPAN_TOKEN_USAGE; + nextConfig.traceFilters = []; + nextConfig.threadFilters = []; + nextConfig.spanFilters = []; + nextConfig.feedbackScores = []; + nextConfig.durationMetrics = []; + nextConfig.usageMetrics = []; + nextConfig.breakdown = { field: BREAKDOWN_FIELD.NONE }; + form.setValue("metricType", METRIC_NAME_TYPE.SPAN_TOKEN_USAGE); + form.setValue("breakdown.field", BREAKDOWN_FIELD.NONE); + form.setValue("traceFilters", []); + form.setValue("threadFilters", []); + form.setValue("spanFilters", []); + form.setValue("feedbackScores", []); + form.setValue("durationMetrics", []); + form.setValue("usageMetrics", []); + } + + updatePreviewWidget({ config: nextConfig }); }; const handleFeedbackScoresChange = (newFeedbackScores: string[]) => { @@ -408,9 +455,9 @@ const ProjectMetricsEditor = forwardRef((_, ref) => {
{ - const validationErrors = get(formState.errors, ["projectId"]); + const validationErrors = get(formState.errors, ["projectIds"]); return ( Project @@ -419,10 +466,13 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { className={cn("flex-1", { "border-destructive": Boolean(validationErrors?.message), })} - value={field.value || ""} + multiselect + showSelectAll + selectAllLabel="All projects" + value={field.value || []} onValueChange={(value) => { field.onChange(value); - handleProjectChange(value); + handleProjectsChange(value); }} disabled={hasRuntimeProjectId} /> @@ -451,7 +501,7 @@ const ProjectMetricsEditor = forwardRef((_, ref) => { field.onChange(value); handleMetricTypeChange(value); }} - options={METRIC_OPTIONS} + options={metricOptions} placeholder="Select a metric type" /> diff --git a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx index c624f3f7667..076d8abc1bb 100644 --- a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx +++ b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/ProjectMetricsWidget.tsx @@ -7,6 +7,10 @@ import { INTERVAL_TYPE, METRIC_NAME_TYPE, } from "@/api/projects/useProjectMetric"; +import { + isWorkspaceMetric, + resolveProjectSelection, +} from "@/lib/dashboard/workspaceMetrics"; import { useDashboardStore, selectRuntimeConfig, @@ -81,22 +85,25 @@ const ProjectMetricsWidget: React.FunctionComponent< }, [sectionId, widgetId, onAddEditWidgetCallback]); const widgetProjectId = widget?.config?.projectId as string | undefined; + const widgetProjectIds = widget?.config?.projectIds as string[] | undefined; + + const { projectId, projectIds } = useMemo( + () => + resolveProjectSelection({ + runtimeProjectId: runtimeContext.projectId, + projectId: widgetProjectId, + projectIds: widgetProjectIds, + }), + [widgetProjectId, widgetProjectIds, runtimeContext.projectId], + ); - const { projectId, interval, intervalStart, intervalEnd } = useMemo(() => { - const resolvedProjectId = - runtimeContext.projectId || widgetProjectId || undefined; - - const { interval, intervalStart, intervalEnd } = calculateIntervalConfig( - runtimeContext.dateRange, - ); + const { interval, intervalStart, intervalEnd } = useMemo( + () => calculateIntervalConfig(runtimeContext.dateRange), + [runtimeContext.dateRange], + ); - return { - projectId: resolvedProjectId, - interval, - intervalStart, - intervalEnd, - }; - }, [widgetProjectId, runtimeContext.projectId, runtimeContext.dateRange]); + const isWorkspaceMode = Array.isArray(projectIds); + const isConfigured = Boolean(projectId) || isWorkspaceMode; const metricType = widget?.config?.metricType as string | undefined; const metricName = metricType as METRIC_NAME_TYPE | undefined; @@ -396,7 +403,7 @@ const ProjectMetricsWidget: React.FunctionComponent< const threadFilters = widget.config?.threadFilters as Filter[] | undefined; const spanFilters = widget.config?.spanFilters as Filter[] | undefined; - if (!projectId) { + if (!isConfigured) { return ( + ); + } + const validTraceFilters = traceFilters?.filter(isFilterValid); const validThreadFilters = threadFilters?.filter(isFilterValid); const validSpanFilters = spanFilters?.filter(isFilterValid); @@ -444,7 +462,8 @@ const ProjectMetricsWidget: React.FunctionComponent< isAggregateTotal={!!isAggregateTotal} intervalStart={intervalStart} intervalEnd={intervalEnd} - projectId={projectId!} + projectId={projectId} + projectIds={projectIds} chartType={chartType} traceFilters={validTraceFilters} threadFilters={validThreadFilters} diff --git a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts index ecc7433fa3a..c3bc0cc18d6 100644 --- a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts +++ b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectMetricsWidget/schema.ts @@ -31,6 +31,7 @@ export const ProjectMetricsWidgetSchema = z.object({ .min(1, { message: "Metric type is required" }), chartType: z.nativeEnum(CHART_TYPE), projectId: z.string().optional(), + projectIds: z.array(z.string()).optional(), traceFilters: FiltersArraySchema.optional(), threadFilters: FiltersArraySchema.optional(), spanFilters: FiltersArraySchema.optional(), diff --git a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx index 8be594d9400..33dc7f01eba 100644 --- a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx +++ b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardEditor.tsx @@ -41,7 +41,15 @@ import { ProjectStatsCardWidgetFormData, } from "./schema"; import { getAllMetricOptions } from "./metrics"; +import { + getWorkspaceStatMetric, + WORKSPACE_STAT_METRIC_OPTIONS, + isMultiProjectSelection, + DEFAULT_WORKSPACE_USAGE_METRIC, +} from "@/lib/dashboard/workspaceMetrics"; import useTracesOrSpansScoresColumns from "@/hooks/useTracesOrSpansScoresColumns"; +import useProjectTokenUsageNames from "@/api/projects/useProjectTokenUsageNames"; +import { METRIC_NAME_TYPE } from "@/api/projects/useProjectMetric"; import { TRACE_DATA_TYPE } from "@/constants/traces"; const ProjectStatsCardEditor = forwardRef((_, ref) => { @@ -53,7 +61,14 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { const { config } = widgetData; const source = config.source || TRACE_DATA_TYPE.traces; const metric = config.metric || ""; + const usageMetric = config.usageMetric || ""; const localProjectId = config.projectId; + const localProjectIds = useMemo( + () => + (config.projectIds as string[] | undefined) ?? + (localProjectId ? [localProjectId] : []), + [config.projectIds, localProjectId], + ); const traceFilters = useMemo( () => config.traceFilters || [], @@ -71,21 +86,51 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }; }); const hasRuntimeProjectId = !!runtimeContext.projectId; - const projectId = runtimeContext.projectId || localProjectId || ""; + // Representative project (runtime, else first selected) for loading option lists. + const projectId = runtimeContext.projectId || localProjectIds[0] || ""; const isTraceSource = source === TRACE_DATA_TYPE.traces; + // Selecting more than one project aggregates across projects, which only supports span total metrics. + const isMultiProject = isMultiProjectSelection( + runtimeContext.projectId, + localProjectIds, + ); + const workspaceMetricDef = isMultiProject + ? getWorkspaceStatMetric(metric) + : null; + const requiresUsageKey = Boolean(workspaceMetricDef?.requiresUsageKey); + // Multi-project aggregates span metrics, so filters are always span filters there; single-project follows source. + const useSpanFilters = isMultiProject || !isTraceSource; + const { data, isPending } = useTracesOrSpansScoresColumns( { projectId, type: source, }, - {}, + { enabled: !isMultiProject && !!projectId }, + ); + + const { data: tokenUsageNamesData } = useProjectTokenUsageNames( + { projectId }, + { enabled: isMultiProject && requiresUsageKey && !!projectId }, + ); + + const usageKeyOptions = useMemo( + () => + (tokenUsageNamesData?.names ?? []) + .slice() + .sort((a, b) => a.localeCompare(b)) + .map((name) => ({ value: name, label: name })), + [tokenUsageNamesData?.names], ); const metricOptions = useMemo(() => { + if (isMultiProject) { + return WORKSPACE_STAT_METRIC_OPTIONS; + } const scoreNames = data?.scores.map((s) => s.name) || []; return getAllMetricOptions(source, scoreNames); - }, [source, data?.scores]); + }, [isMultiProject, source, data?.scores]); const form = useForm({ resolver: zodResolver(ProjectStatsCardWidgetSchema), @@ -93,7 +138,8 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { defaultValues: { source, metric, - projectId, + projectIds: localProjectIds, + usageMetric, traceFilters, spanFilters, }, @@ -126,20 +172,50 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }); }; - const handleProjectChange = (projectId: string) => { - updatePreviewWidget({ - config: { - ...config, - projectId, - }, - }); + const handleProjectsChange = (projectIds: string[]) => { + const nextConfig = { ...config, projectIds }; + delete nextConfig.projectId; + form.setValue("projectIds", projectIds); + + const becomesMultiProject = !hasRuntimeProjectId && projectIds.length >= 2; + if (becomesMultiProject && !getWorkspaceStatMetric(metric)) { + // Switching into multi-project with an incompatible metric: default to span token usage total. + nextConfig.metric = METRIC_NAME_TYPE.SPAN_TOKEN_USAGE; + nextConfig.usageMetric = DEFAULT_WORKSPACE_USAGE_METRIC; + nextConfig.traceFilters = []; + nextConfig.spanFilters = []; + form.setValue("metric", METRIC_NAME_TYPE.SPAN_TOKEN_USAGE); + form.setValue("usageMetric", DEFAULT_WORKSPACE_USAGE_METRIC); + form.setValue("traceFilters", []); + form.setValue("spanFilters", []); + } else if (!becomesMultiProject && getWorkspaceStatMetric(metric)) { + // Leaving multi-project with a workspace-only metric selected: clear it so the user picks a valid one. + nextConfig.metric = ""; + form.setValue("metric", ""); + } + + updatePreviewWidget({ config: nextConfig }); }; const handleMetricChange = (value: string) => { + const nextConfig = { ...config, metric: value }; + // Picking the token-usage total needs a usage key; default it. + if ( + isMultiProject && + getWorkspaceStatMetric(value)?.requiresUsageKey && + !config.usageMetric + ) { + nextConfig.usageMetric = DEFAULT_WORKSPACE_USAGE_METRIC; + form.setValue("usageMetric", DEFAULT_WORKSPACE_USAGE_METRIC); + } + updatePreviewWidget({ config: nextConfig }); + }; + + const handleUsageMetricChange = (value: string) => { updatePreviewWidget({ config: { ...config, - metric: value, + usageMetric: value, }, }); }; @@ -149,9 +225,9 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => {
{ - const validationErrors = get(formState.errors, ["projectId"]); + const validationErrors = get(formState.errors, ["projectIds"]); return ( Project @@ -160,10 +236,13 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { className={cn("flex-1", { "border-destructive": Boolean(validationErrors?.message), })} - value={field.value || ""} + multiselect + showSelectAll + selectAllLabel="All projects" + value={field.value || []} onValueChange={(value) => { field.onChange(value); - handleProjectChange(value); + handleProjectsChange(value); }} disabled={hasRuntimeProjectId} /> @@ -174,28 +253,66 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }} /> + {!isMultiProject && ( + { + const validationErrors = get(formState.errors, ["source"]); + return ( + + Source + + { + field.onChange(value); + handleSourceChange(value); + }} + options={SOURCE_OPTIONS} + placeholder="Select source" + renderOption={renderSourceOption} + renderTrigger={renderSourceTrigger} + /> + + + + ); + }} + /> + )} + { - const validationErrors = get(formState.errors, ["source"]); + const validationErrors = get(formState.errors, ["metric"]); + const placeholder = + !isMultiProject && isPending + ? "Loading available metrics..." + : "Select a metric"; + return ( - Source + Metric type - { field.onChange(value); - handleSourceChange(value); + handleMetricChange(value); }} - options={SOURCE_OPTIONS} - placeholder="Select source" - renderOption={renderSourceOption} - renderTrigger={renderSourceTrigger} + options={metricOptions} + placeholder={placeholder} + disabled={!isMultiProject && isPending} /> @@ -204,51 +321,42 @@ const ProjectStatsCardEditor = forwardRef((_, ref) => { }} /> - { - const validationErrors = get(formState.errors, ["metric"]); - const placeholder = isPending - ? "Loading available metrics..." - : "Select a metric"; - - return ( + {isMultiProject && requiresUsageKey && ( + ( - Metric type + Usage metric { field.onChange(value); - handleMetricChange(value); + handleUsageMetricChange(value); }} - options={metricOptions} - placeholder={placeholder} - disabled={isPending} + options={usageKeyOptions} + placeholder="Select a usage metric" /> - ); - }} - /> + )} + /> + )} { updatePreviewWidget({ config: { ...config, - ...(isTraceSource - ? { traceFilters: filters } - : { spanFilters: filters }), + ...(useSpanFilters + ? { spanFilters: filters } + : { traceFilters: filters }), }, }); }} diff --git a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx index 1f49a65dc08..21a91fa77fc 100644 --- a/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx +++ b/apps/opik-frontend/src/v2/pages-shared/dashboards/widgets/ProjectStatsCardWidget/ProjectStatsCardWidget.tsx @@ -15,6 +15,11 @@ import { calculateIntervalConfig } from "@/v2/pages-shared/traces/MetricDateRang import { DEFAULT_DATE_PRESET } from "@/v2/pages-shared/traces/MetricDateRangeSelect/constants"; import useTracesStatistic from "@/api/traces/useTracesStatistic"; import useSpansStatistic from "@/api/traces/useSpansStatistic"; +import useWorkspaceMetric from "@/api/projects/useWorkspaceMetric"; +import { + INTERVAL_TYPE, + METRIC_NAME_TYPE, +} from "@/api/projects/useProjectMetric"; import { ColumnStatistic } from "@/types/shared"; import { Skeleton } from "@/ui/skeleton"; import { TRACE_DATA_TYPE } from "@/constants/traces"; @@ -26,6 +31,11 @@ import { isFeedbackScoreMetric, extractFeedbackScoreName, } from "./metrics"; +import { + getWorkspaceStatMetric, + resolveProjectSelection, + DEFAULT_WORKSPACE_USAGE_METRIC, +} from "@/lib/dashboard/workspaceMetrics"; import { formatScoreDisplay } from "@/lib/feedback-scores"; const renderMetricDisplay = ( @@ -81,24 +91,32 @@ const ProjectStatsCardWidget: React.FunctionComponent< }, [sectionId, widgetId, onAddEditWidgetCallback]); const widgetProjectId = widget?.config?.projectId as string | undefined; + const widgetProjectIds = widget?.config?.projectIds as string[] | undefined; - const { projectId, intervalStart, intervalEnd } = useMemo(() => { - const resolvedProjectId = - runtimeContext.projectId || widgetProjectId || undefined; + const { projectId, projectIds } = useMemo( + () => + resolveProjectSelection({ + runtimeProjectId: runtimeContext.projectId, + projectId: widgetProjectId, + projectIds: widgetProjectIds, + }), + [widgetProjectId, widgetProjectIds, runtimeContext.projectId], + ); - const { intervalStart, intervalEnd } = calculateIntervalConfig( - runtimeContext.dateRange, - ); + const { intervalStart, intervalEnd } = useMemo( + () => calculateIntervalConfig(runtimeContext.dateRange), + [runtimeContext.dateRange], + ); - return { - projectId: resolvedProjectId, - intervalStart, - intervalEnd, - }; - }, [widgetProjectId, runtimeContext.projectId, runtimeContext.dateRange]); + const isMultiProject = Array.isArray(projectIds); + const isConfigured = Boolean(projectId) || isMultiProject; const source = widget?.config?.source as TRACE_DATA_TYPE | undefined; const metric = widget?.config?.metric as string | undefined; + const usageMetric = widget?.config?.usageMetric as string | undefined; + const workspaceMetricDef = isMultiProject + ? getWorkspaceStatMetric(metric ?? "") + : null; const traceFilters = widget?.config?.traceFilters as Filter[] | undefined; const spanFilters = widget?.config?.spanFilters as Filter[] | undefined; @@ -119,7 +137,10 @@ const ProjectStatsCardWidget: React.FunctionComponent< toTime: intervalEnd, logsSource: LOGS_SOURCE.sdk, }, - { enabled: source === TRACE_DATA_TYPE.traces && !!projectId }, + { + enabled: + !isMultiProject && source === TRACE_DATA_TYPE.traces && !!projectId, + }, ); const spansStatistic = useSpansStatistic( @@ -130,7 +151,22 @@ const ProjectStatsCardWidget: React.FunctionComponent< toTime: intervalEnd, logsSource: LOGS_SOURCE.sdk, }, - { enabled: source === TRACE_DATA_TYPE.spans && !!projectId }, + { + enabled: + !isMultiProject && source === TRACE_DATA_TYPE.spans && !!projectId, + }, + ); + + const workspaceMetric = useWorkspaceMetric( + { + projectIds: projectIds ?? [], + metricName: (metric as METRIC_NAME_TYPE) ?? METRIC_NAME_TYPE.SPAN_COUNT, + interval: INTERVAL_TYPE.TOTAL, + intervalStart, + intervalEnd, + spanFilters: validSpanFilters, + }, + { enabled: isMultiProject && Boolean(workspaceMetricDef) }, ); const { data, isLoading, error } = useMemo( @@ -151,8 +187,50 @@ const ProjectStatsCardWidget: React.FunctionComponent< /> ) : undefined; + const renderMultiProjectContent = () => { + if (!metric || !workspaceMetricDef) { + return ( + + ); + } + + if (workspaceMetric.isLoading) { + return ( +
+
+ + +
+
+ ); + } + + const results = workspaceMetric.data?.results ?? []; + // Token usage returns one series per usage key; pick the selected key. Count/cost return a single series. + const series = workspaceMetricDef.requiresUsageKey + ? results.find( + (r) => r.name === (usageMetric || DEFAULT_WORKSPACE_USAGE_METRIC), + ) + : results[0]; + const value = series?.data?.[0]?.value; + + return renderMetricDisplay( + workspaceMetricDef.label, + value !== undefined && value !== null + ? workspaceMetricDef.formatter(Number(value)) + : "-", + value !== undefined && value !== null + ? workspaceMetricDef.tooltipFormatter?.(Number(value)) + : undefined, + ); + }; + const renderCardContent = () => { - if (!projectId) { + if (!isConfigured) { return (