feat: centralize deduped public API error alerts (CM-1349)

.github/scripts/notify-api-e2e-failure.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${SLACK_WEBHOOK_URL:?}"
+: "${RUN_URL:?}"
+: "${EVENT_NAME:?}"
+: "${ALERT_TITLE:?}"
+
+failed_step() {
+  local pairs=(
+    "${OUTCOME_RESOLVE_TAG}:Resolve deploy image tag"
+    "${OUTCOME_DEPLOY}:Deploy api-e2e"
+    "${OUTCOME_HEALTH}:Wait for api-e2e service to be ready"
+    "${OUTCOME_E2E}:Run e2e tests"
+  )
+  local pair outcome label
+  for pair in "${pairs[@]}"; do
+    outcome="${pair%%:*}"
+    label="${pair#*:}"
+    if [[ "$outcome" == "failure" ]]; then
+      printf '%s\n' "$label"
+      return
+    fi
+  done
+  printf 'unknown\n'
+}
+
+read_summary() {
+  local path="${E2E_SUMMARY_PATH:-e2e.summary}"
+  results=""
+  failures=""
+
+  [[ -f "$path" ]] || return 0
+
+  local passed failed
+  passed="$(awk -F= '/^passed=/{print $2; exit}' "$path")"
+  failed="$(awk -F= '/^failed=/{print $2; exit}' "$path")"
+  if [[ -n "${passed}" || -n "${failed}" ]]; then
+    results="Passed: ${passed:-?} / Failed: ${failed:-?}"
+  fi
+  failures="$(awk '/^FAIL /{print; if (++n == 10) exit}' "$path")"
+}
+
+sha_short="${DEPLOY_TAG:-unknown}"
+sha_short="${sha_short:0:12}"
+
+read_summary
+
+lines=(
+  ":rotating_light: *${ALERT_TITLE}*"
+  "*Event:* \`${EVENT_NAME}\`"
+  "*Failed step:* \`$(failed_step)\`"
+  "*Deploy SHA:* \`${sha_short}\`"
+  "*Run:* <${RUN_URL}|View run>"
+)
+
+[[ -n "${results}" ]] && lines+=("*Results:* ${results}")
+
+if [[ -n "${failures}" ]]; then
+  lines+=("*Failed tests:*" $'```\n'"${failures}"$'\n```')
+fi
+
+text="$(printf '%s\n' "${lines[@]}")"
+payload="$(jq -n --arg text "$text" '{text: $text}')"
+
+curl -fsS -X POST "$SLACK_WEBHOOK_URL" \
+  -H 'Content-Type: application/json' \
+  -d "$payload"

.github/scripts/public-api-e2e-tests.sh

@@ -47,6 +47,7 @@ SOURCE="lfxOne-api-e2e"
 
 PASS=0
 FAIL=0
+FAILURE_LINES=()
 TOKEN=""
 HTTP_CODE=""
 BODY=""
@@ -81,19 +82,25 @@ log_fail_body() {
   printf '  body: %s\n' "$snippet"
 }
 
+record_failure() {
+  local line=$1
+  FAILURE_LINES+=("$line")
+  printf '  %s\n' "$line"
+}
+
 check() {
   local name=$1 expected=$2
   shift 2
   if [[ $HTTP_CODE != "$expected" ]]; then
-    printf '  FAIL  %s — expected HTTP %s, got %s\n' "$name" "$expected" "$HTTP_CODE"
+    record_failure "FAIL  ${name} — expected HTTP ${expected}, got ${HTTP_CODE}"
     log_fail_body
     FAIL=$((FAIL + 1))
     return
   fi
   local expr
   for expr in "$@"; do
     if ! jq -e "$expr" >/dev/null 2>&1 <<<"$BODY"; then
-      printf '  FAIL  %s (%s)\n' "$name" "$expr"
+      record_failure "FAIL  ${name} (${expr})"
       log_fail_body
       FAIL=$((FAIL + 1))
       return
@@ -103,6 +110,16 @@ check() {
   PASS=$((PASS + 1))
 }
 
+write_summary() {
+  local path="${E2E_SUMMARY_PATH:-}"
+  [[ -n "$path" ]] || return 0
+  {
+    printf 'passed=%s\n' "$PASS"
+    printf 'failed=%s\n' "$FAIL"
+    printf '%s\n' "${FAILURE_LINES[@]+"${FAILURE_LINES[@]}"}"
+  } >"$path"
+}
+
 require() {
   local expected=$1 name=$2
   [[ $HTTP_CODE == "$expected" ]] || die "$name — expected HTTP $expected, got $HTTP_CODE"
@@ -502,6 +519,7 @@ main() {
   echo "=== Results ==="
   printf 'Passed: %s\n' "$PASS"
   printf 'Failed: %s\n' "$FAIL"
+  write_summary
   [[ $FAIL -eq 0 ]]
 }
 

.github/workflows/api-e2e-tests.yml

@@ -25,7 +25,7 @@ jobs:
           ref: main
 
       - name: Resolve deploy image tag
-        id: deploy
+        id: resolve_tag
         run: echo "tag=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Install OCI
@@ -43,6 +43,7 @@ jobs:
         run: echo "${{ env.OCI_CLI_DIR }}" >> "$GITHUB_PATH"
 
       - name: Deploy api-e2e
+        id: deploy
         uses: ./.github/actions/node/builder
         env:
           CLOUD_ENV: lf-oracle-staging
@@ -56,9 +57,10 @@ jobs:
           ORACLE_CLUSTER: ${{ secrets.ORACLE_STAGING_CLUSTER }}
         with:
           services: api-e2e
-          tag: ${{ steps.deploy.outputs.tag }}
+          tag: ${{ steps.resolve_tag.outputs.tag }}
 
       - name: Wait for api-e2e service to be ready
+        id: health
         run: |
           set -euo pipefail
           # Builder only runs kubectl set image; wait for the new revision before health.
@@ -72,43 +74,26 @@ jobs:
           exit 1
 
       - name: Run Public API e2e tests
+        id: e2e
         env:
           AUTH0_STAGING_AUDIENCE: ${{ vars.AUTH0_STAGING_AUDIENCE }}
           AUTH0_STAGING_ISSUER: ${{ vars.AUTH0_STAGING_ISSUER }}
           AUTH0_STAGING_API_E2E_CLIENT_ID: ${{ secrets.AUTH0_STAGING_API_E2E_CLIENT_ID }}
           AUTH0_STAGING_API_E2E_CLIENT_SECRET: ${{ secrets.AUTH0_STAGING_API_E2E_CLIENT_SECRET }}
+          E2E_SUMMARY_PATH: e2e.summary
         run: bash .github/scripts/public-api-e2e-tests.sh
 
       - name: Notify Slack on failure
-        if: failure()
+        if: failure() && !cancelled()
         env:
-          CDP_ALERTS_SLACK_WEBHOOK_URL: ${{ secrets.CDP_ALERTS_SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_URL: ${{ secrets.CDP_ALERTS_SLACK_WEBHOOK_URL }}
+          ALERT_TITLE: Public API e2e tests failed
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
           EVENT_NAME: ${{ github.event_name }}
-        run: |
-          set -euo pipefail
-          payload="$(jq -n \
-            --arg run_url "$RUN_URL" \
-            --arg event "$EVENT_NAME" \
-            '{
-              blocks: [
-                {
-                  type: "header",
-                  text: {
-                    type: "plain_text",
-                    text: ":rotating_light: Public API e2e tests failed",
-                    emoji: true
-                  }
-                },
-                {
-                  type: "section",
-                  text: {
-                    type: "mrkdwn",
-                    text: ("*Workflow:* `API e2e tests`\n*Event:* `" + $event + "`\n*Run:* <" + $run_url + "|View run>")
-                  }
-                }
-              ]
-            }')"
-          curl -fsS -X POST "$CDP_ALERTS_SLACK_WEBHOOK_URL" \
-            -H 'Content-Type: application/json' \
-            -d "$payload"
+          DEPLOY_TAG: ${{ steps.resolve_tag.outputs.tag }}
+          E2E_SUMMARY_PATH: e2e.summary
+          OUTCOME_RESOLVE_TAG: ${{ steps.resolve_tag.outcome }}
+          OUTCOME_DEPLOY: ${{ steps.deploy.outcome }}
+          OUTCOME_HEALTH: ${{ steps.health.outcome }}
+          OUTCOME_E2E: ${{ steps.e2e.outcome }}
+        run: bash .github/scripts/notify-api-e2e-failure.sh

backend/src/api/public/alerts/alertOnce.ts

@@ -0,0 +1,111 @@
+import { createHash } from 'crypto'
+import type { Request } from 'express'
+
+import { generateUUIDv4 } from '@crowd/common'
+import { RedisCache } from '@crowd/redis'
+import {
+  SlackChannel,
+  type SlackMessageSection,
+  SlackPersona,
+  sendSlackNotification,
+} from '@crowd/slack'
+
+const PATH_UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}/gi
+
+export async function alertOnce(
+  req: Request,
+  {
+    status,
+    code,
+    message,
+    name,
+    context,
+    stack,
+  }: {
+    status: number
+    code: string
+    message: string
+    name?: string
+    context?: Record<string, unknown>
+    stack?: string
+  },
+): Promise<void> {
+  if (status !== 409 && status < 500) return
+
+  const path = (req.originalUrl || req.url || '').split('?')[0]
+
+  // akrites alerts are handled separately, so skip them here.
+  if (path.startsWith('/v1/akrites') || path.startsWith('/v1/akrites-external')) {
+    return
+  }
+
+  const route = resolveRoute(req)
+
+  const dedupeKey = createHash('sha256')
+    .update(
+      [status, req.method, route, code, message, serializeContext(context)]
+        .filter((part) => part !== '')
+        .join(':'),
+    )
+    .digest('hex')
+
+  const cache = new RedisCache('public-api-alerts', req.redis, req.log)
+  const lease = generateUUIDv4()
+
+  try {
+    const held = await cache.setIfNotExistsOrGet(dedupeKey, lease, 60 * 60)
+    if (held !== lease) {
+      req.log.info({ dedupeKey }, 'Skipping duplicate public API alert')
+      return
+    }
+  } catch (err) {
+    req.log.warn({ err, dedupeKey }, 'Alert dedupe failed; sending anyway')
+  }
+
+  const sections: SlackMessageSection[] = [
+    {
+      title: 'Request',
+      text: `*Method:* \`${req.method}\`\n*URL:* \`${req.originalUrl || req.url}\``,
+    },
+    {
+      title: 'Error',
+      text: `*Code:* \`${code}\`\n*Name:* \`${name || code}\`\n*Message:* ${message}`,
+    },
+  ]
+
+  if (context && Object.keys(context).length > 0) {
+    sections.push({
+      title: 'Context',
+      text: `\`\`\`${JSON.stringify(context, null, 2)}\`\`\``,
+    })
+  }
+
+  if (stack) {
+    sections.push({
+      title: 'Stack Trace',
+      text: `\`\`\`${stack.substring(0, 2700)}\`\`\``,
+    })
+  }
+
+  sendSlackNotification(
+    SlackChannel.CDP_PUBLIC_API_ALERTS,
+    status >= 500 ? SlackPersona.ERROR_REPORTER : SlackPersona.WARNING_PROPAGATOR,
+    status >= 500 ? `500 Error: ${name || message}` : `${status} Conflict: ${message}`,
+    sections,
+  )
+}
+
+function resolveRoute(req: Request): string {
+  return (req.originalUrl || req.url || '').split('?')[0].replace(PATH_UUID, ':id')
+}
+
+function serializeContext(context?: Record<string, unknown>): string {
+  if (!context) return ''
+
+  const normalized: Record<string, unknown> = {}
+  for (const key of Object.keys(context).sort()) {
+    const value = context[key]
+    normalized[key] = Array.isArray(value) ? [...value].map(String).sort() : value
+  }
+  return JSON.stringify(normalized)
+}