From 3c1c19970e1d1811d323f5ee89bba91ad09ab4e0 Mon Sep 17 00:00:00 2001 From: danilo neves cruz Date: Tue, 21 Jul 2026 22:33:01 +0200 Subject: [PATCH 1/3] =?UTF-8?q?=E2=9C=A8=20server:=20process=20auto-credit?= =?UTF-8?q?=20in=20worker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/afraid-mangos-follow.md | 5 + .changeset/tender-foxes-feel.md | 5 + infra/utils/modules.ts | 1 + server/api/card.ts | 22 +- server/api/index.ts | 5 +- server/hooks/activity.ts | 42 +--- server/i18n/es.json | 4 +- server/i18n/pt.json | 4 +- server/index.ts | 5 + server/script/openapi.ts | 1 + server/test/api/card.test.ts | 87 ++++--- server/test/e2e.ts | 3 +- server/test/hooks/activity.test.ts | 160 +------------ server/test/workers/bin.test.ts | 24 ++ server/test/workers/credit.test.ts | 351 +++++++++++++++++++++++++++++ server/utils/panda.ts | 26 +-- server/workers/credit/bin.ts | 28 +++ server/workers/credit/job.ts | 10 + server/workers/credit/queue.ts | 14 ++ server/workers/credit/worker.ts | 84 +++++++ 20 files changed, 626 insertions(+), 255 deletions(-) create mode 100644 .changeset/afraid-mangos-follow.md create mode 100644 .changeset/tender-foxes-feel.md create mode 100644 server/test/workers/credit.test.ts create mode 100644 server/workers/credit/bin.ts create mode 100644 server/workers/credit/job.ts create mode 100644 server/workers/credit/queue.ts create mode 100644 server/workers/credit/worker.ts diff --git a/.changeset/afraid-mangos-follow.md b/.changeset/afraid-mangos-follow.md new file mode 100644 index 0000000000..e5443de564 --- /dev/null +++ b/.changeset/afraid-mangos-follow.md @@ -0,0 +1,5 @@ +--- +"@exactly/infra": minor +--- + +🧱 setup credit queue worker diff --git a/.changeset/tender-foxes-feel.md b/.changeset/tender-foxes-feel.md new file mode 100644 index 0000000000..ca54e85e86 --- /dev/null +++ b/.changeset/tender-foxes-feel.md @@ -0,0 +1,5 @@ +--- +"@exactly/server": patch +--- + +✨ process auto-credit in worker diff --git a/infra/utils/modules.ts b/infra/utils/modules.ts index 55f40833a1..7fab61989e 100644 --- a/infra/utils/modules.ts +++ b/infra/utils/modules.ts @@ -12,6 +12,7 @@ export default define({ env: { WHATSAPP_PHONE_NUMBER_ID: "whatsappPhoneNumberId" }, secrets: ["anthropic-api-key", "whatsapp-access-token"], }, + credit: { secrets: ["onesignal-api-key", "postgres-url"] }, hook: { secrets: ["panda-api-key", "postgres-url"], shared: ["panda-api-url"] }, refund: { secrets: ["panda-api-key", "onesignal-api-key", "postgres-url", "sardine-api-key", "segment-write-key"], diff --git a/server/api/card.ts b/server/api/card.ts index 911f5b7290..e3e31831e6 100644 --- a/server/api/card.ts +++ b/server/api/card.ts @@ -40,10 +40,10 @@ import { BASE_PRODUCT_ID, PLATINUM_PRODUCT_ID, SIGNATURE_PRODUCT_ID } from "@exa import { Address, Base64URL, Hex } from "@exactly/common/validation"; import { cards, credentials } from "../database/schema"; -import { autoCredit } from "../utils/panda"; import publicClient from "../utils/publicClient"; import ServiceError from "../utils/ServiceError"; import validatorHook from "../utils/validatorHook"; +import { name as creditName } from "../workers/credit/job"; import type * as schema from "../database/schema"; import type { Auth } from "../middleware/auth"; @@ -53,6 +53,7 @@ import type createPersona from "../utils/persona"; import type createSardine from "../utils/sardine"; import type createSegment from "../utils/segment"; import type createWalletExtension from "../utils/walletExtension"; +import type createCredit from "../workers/credit/queue"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; const CardResponse = object({ @@ -150,6 +151,7 @@ const Scopes = picklist(["provisioning", "siwe", "webauthn"]); export default function route({ auth, + credit, database, panda, pax, @@ -159,6 +161,7 @@ export default function route({ walletExtension, }: { auth: Auth; + credit: ReturnType; database: NodePgDatabase; panda: ReturnType; pax: ReturnType; @@ -644,15 +647,14 @@ This endpoint only accepts Wallet Extension bearer access. It does not accept \` } }); - let mode = 0; - try { - if (await autoCredit(account)) mode = 1; - } catch (error) { - captureException(error); - } - await database - .insert(cards) - .values([{ id: card.id, credentialId, lastFour: card.last4, mode, productId }]); + await database.insert(cards).values([{ id: card.id, credentialId, lastFour: card.last4, productId }]); + await credit.enqueue(account).catch((error: unknown) => + captureException(error, { + level: "error", + tags: { queue: creditName, job: creditName }, + extra: { account }, + }), + ); segment.track({ event: "CardIssued", userId: account, diff --git a/server/api/index.ts b/server/api/index.ts index 08c9260d35..fb9e28ef8f 100644 --- a/server/api/index.ts +++ b/server/api/index.ts @@ -27,6 +27,7 @@ import type createManteca from "../utils/ramps/manteca"; import type createSardine from "../utils/sardine"; import type createSegment from "../utils/segment"; import type createWalletExtension from "../utils/walletExtension"; +import type createCredit from "../workers/credit/queue"; import type createSubscribe from "../workers/subscribe/queue"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Redis } from "ioredis"; @@ -34,6 +35,7 @@ import type { Redis } from "ioredis"; export default function api({ authSecret, bridge, + credit, database, intercom, manteca, @@ -48,6 +50,7 @@ export default function api({ }: { authSecret: string; bridge: ReturnType; + credit: ReturnType; database: NodePgDatabase; intercom: ReturnType; manteca: ReturnType; @@ -77,7 +80,7 @@ export default function api({ authentication({ authSecret, createCredential: credential, database, intercom, redis, walletExtension }), ) .route("/activity", activity({ auth, database })) - .route("/card", card({ auth, database, panda, pax, persona, sardine, segment, walletExtension })) + .route("/card", card({ auth, credit, database, panda, pax, persona, sardine, segment, walletExtension })) .route("/kyc", kyc({ auth, database, panda, persona })) .route("/passkey", passkey({ auth, database })) // eslint-disable-line @typescript-eslint/no-deprecated -- // TODO remove .route("/pax", paxRoute({ auth, database, pax })) diff --git a/server/hooks/activity.ts b/server/hooks/activity.ts index 22740dfcff..bd57b5be86 100644 --- a/server/hooks/activity.ts +++ b/server/hooks/activity.ts @@ -13,7 +13,7 @@ import { withScope, } from "@sentry/node"; import createDebug from "debug"; -import { eq, inArray } from "drizzle-orm"; +import { inArray } from "drizzle-orm"; import { Hono } from "hono"; import { validator } from "hono/validator"; import * as v from "valibot"; @@ -32,11 +32,10 @@ import exaChain, { } from "@exactly/common/generated/chain"; import { Address, Hash, Hex } from "@exactly/common/validation"; -import { cards, credentials } from "../database/schema"; +import { credentials } from "../database/schema"; import t, { f } from "../i18n"; import { activityNetworks, activityUrl, NETWORKS } from "../utils/alchemy"; import decodePublicKey from "../utils/decodePublicKey"; -import { autoCredit } from "../utils/panda"; import publicClient from "../utils/publicClient"; import revertFingerprint from "../utils/revertFingerprint"; import validatorHook from "../utils/validatorHook"; @@ -47,6 +46,7 @@ import type * as schema from "../database/schema"; import type createAlchemy from "../utils/alchemy"; import type createOnesignal from "../utils/onesignal"; import type createSegment from "../utils/segment"; +import type createCredit from "../workers/credit/queue"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Redis } from "ioredis"; @@ -58,6 +58,7 @@ Object.assign(debug, { inspectOpts: { depth: undefined } }); export default function hook({ alchemy, + credit, database, executor, onesignal, @@ -65,6 +66,7 @@ export default function hook({ segment, }: { alchemy: ReturnType; + credit: ReturnType; database: NodePgDatabase; executor: LocalAccount; onesignal: ReturnType; @@ -280,7 +282,10 @@ export default function hook({ ), ); for (const result of results) { - if (result.status === "fulfilled") continue; + if (result.status === "fulfilled") { + await credit.enqueue(account); + continue; + } if (result.reason instanceof Error && result.reason.message === "NoBalance()") { withScope((captureScope) => { captureScope.setUser({ id: account }); @@ -298,35 +303,6 @@ export default function hook({ span.setStatus({ code: SPAN_STATUS_ERROR, message: "poke_failed" }); throw result.reason; } - autoCredit(account) - .then(async (auto) => { - span.setAttribute("exa.autoCredit", auto); - if (!auto) return; - const credential = await database.query.credentials.findFirst({ - where: eq(credentials.account, account), - columns: {}, - with: { - cards: { - columns: { id: true, mode: true }, - where: inArray(cards.status, ["ACTIVE", "FROZEN"]), - }, - }, - }); - const card = credential?.cards[0]; - if (!card) return; - span.setAttribute("exa.card", card.id); - if (card.mode !== 0) return; - await database.update(cards).set({ mode: 1 }).where(eq(cards.id, card.id)); - span.setAttribute("exa.mode", 1); - onesignal - .sendPushNotification({ - userId: account, - headings: t("Card mode changed"), - contents: t("Credit mode activated"), - }) - .catch((error: unknown) => captureException(error)); - }) - .catch((error: unknown) => captureException(error)); span.setStatus({ code: SPAN_STATUS_OK }); }, ), diff --git a/server/i18n/es.json b/server/i18n/es.json index f02c34f3fa..74e08c5e11 100644 --- a/server/i18n/es.json +++ b/server/i18n/es.json @@ -2,11 +2,8 @@ "atm limit reached. maximum 250 usd per transaction.": "límite de cajero alcanzado. máximo 250 usd por transacción.", "advertising limit reached": "se ha alcanzado el límite de compras de publicidad", "atm limit reached": "se ha alcanzado el límite de retiros en cajero", - "Card mode": "Modo de pago", - "Card mode changed": "Modo de pago actualizado", "Card purchase": "Compra con tarjeta", "Credit mode activated": "Modo crédito activado", - "Credit mode is active": "El modo crédito está activo", "Deposited funds": "Fondos depositados", "Exa Card purchase rejected": "Compra con Exa Card rechazada", "Fiat onramp activated": "Cuenta fiat activada", @@ -16,6 +13,7 @@ "Transaction at {{merchantName}} for {{amount}} rejected: {{reason}}": "Transacción en {{merchantName}} por {{amount}} rechazada: {{reason}}", "Withdraw completed": "Retiro completado", "Withdrawal in progress": "Retiro en curso", + "Your card is now in credit mode": "Tu tarjeta ahora está en modo crédito", "Your debt is due in 1 hour. Repay now to avoid penalties.": "Tu deuda vence en 1 hora. Repágala ahora para evitar penalidades.", "Your debt is due in 24 hours. Repay now to avoid penalties.": "Tu deuda vence en 24 horas. Repágala ahora para evitar penalidades.", "Your fiat onramp account has been activated": "Tu cuenta para depositar dinero fiat ha sido activada", diff --git a/server/i18n/pt.json b/server/i18n/pt.json index 1831db3ed0..750b9ccb90 100644 --- a/server/i18n/pt.json +++ b/server/i18n/pt.json @@ -2,11 +2,8 @@ "atm limit reached. maximum 250 usd per transaction.": "limite de saque atingido. máximo de 250 usd por transação.", "advertising limit reached": "limite de publicidade atingido", "atm limit reached": "limite de saque atingido", - "Card mode": "Modo de pagamento", - "Card mode changed": "Modo de pagamento atualizado", "Card purchase": "Compra no cartão", "Credit mode activated": "Modo crédito ativado", - "Credit mode is active": "O modo crédito está ativo", "Deposited funds": "Fundos depositados", "Exa Card purchase rejected": "Compra com Exa Card recusada", "Fiat onramp activated": "Conta fiat ativada", @@ -16,6 +13,7 @@ "Transaction at {{merchantName}} for {{amount}} rejected: {{reason}}": "Transação em {{merchantName}} de {{amount}} recusada: {{reason}}", "Withdraw completed": "Saque concluído", "Withdrawal in progress": "Saque em andamento", + "Your card is now in credit mode": "Seu cartão agora está no modo crédito", "Your debt is due in 1 hour. Repay now to avoid penalties.": "Sua dívida vence em 1 hora. Pague agora para evitar penalidades.", "Your debt is due in 24 hours. Repay now to avoid penalties.": "Sua dívida vence em 24 horas. Pague agora para evitar penalidades.", "Your fiat onramp account has been activated": "Sua conta para depositar dinheiro fiat foi ativada", diff --git a/server/index.ts b/server/index.ts index b9fcf40237..a7c40dca00 100644 --- a/server/index.ts +++ b/server/index.ts @@ -36,6 +36,7 @@ import createSardine from "./utils/sardine"; import createSegment from "./utils/segment"; import { legacy } from "./utils/wallet"; import createWalletExtension from "./utils/walletExtension"; +import createCredit from "./workers/credit/queue"; import createHook from "./workers/hook/queue"; import createHookWorker from "./workers/hook/worker"; import createRefund from "./workers/refund/queue"; @@ -67,6 +68,7 @@ const persona = createPersona( parse(pipe(string("persona key"), nonEmpty("persona key")), env.PERSONA_API_KEY), parse(pipe(string("persona url"), nonEmpty("persona url")), env.PERSONA_URL), ); +const credit = createCredit(redis); const refund = createRefund(redis); const webhook = createHook(redis); const sardine = createSardine( @@ -82,6 +84,7 @@ setupMaturity(onesignal); const api = createApi({ authSecret: parse(pipe(string("auth"), nonEmpty("auth")), env.AUTH_SECRET), bridge, + credit, database, intercom, manteca, @@ -97,6 +100,7 @@ const api = createApi({ const activityHook = createActivityHook({ alchemy, + credit, database, executor: keeper, onesignal, @@ -416,6 +420,7 @@ export const close = supervise( ]), }, closeMaturity, + () => credit.close(), () => hookWorker.close(), () => refund.close(), () => segment.close(), diff --git a/server/script/openapi.ts b/server/script/openapi.ts index 92063835b8..cfeca5eac2 100644 --- a/server/script/openapi.ts +++ b/server/script/openapi.ts @@ -25,6 +25,7 @@ import("../api") const handle = api({ authSecret: zeroHash, bridge: createBridge("bridge", "https://bridge.test"), + credit: { close: () => Promise.resolve(), enqueue: () => Promise.resolve() }, database, intercom: createIntercom("intercom"), manteca: createManteca("manteca", "https://manteca.test"), diff --git a/server/test/api/card.test.ts b/server/test/api/card.test.ts index 221cf9aa2e..b024a47128 100644 --- a/server/test/api/card.test.ts +++ b/server/test/api/card.test.ts @@ -19,7 +19,7 @@ import { createSecretKey } from "node:crypto"; import { env } from "node:process"; import { nonEmpty, parse, pipe, string } from "valibot"; import { checksumAddress, hexToBigInt, padHex, parseEther, zeroHash } from "viem"; -import { privateKeyToAccount, privateKeyToAddress } from "viem/accounts"; +import { generatePrivateKey, privateKeyToAccount, privateKeyToAddress } from "viem/accounts"; import { base, optimism } from "viem/chains"; import { createSiweMessage, parseSiweMessage } from "viem/siwe"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, inject, it, vi } from "vitest"; @@ -35,7 +35,7 @@ import database, { cards, credentials } from "../../database"; import authenticate from "../../middleware/auth"; import createAuth from "../../utils/auth"; import authSecret from "../../utils/authSecret"; -import createPanda, * as Panda from "../../utils/panda"; +import createPanda from "../../utils/panda"; import createPax from "../../utils/pax"; import createPersona from "../../utils/persona"; import createSardine from "../../utils/sardine"; @@ -44,6 +44,7 @@ import ServiceError from "../../utils/ServiceError"; import wallet, { signer } from "../../utils/wallet"; import createWalletExtension from "../../utils/walletExtension"; +import type createCredit from "../../workers/credit/queue"; import type * as sentry from "@sentry/node"; import type { UnofficialStatusCode } from "hono/utils/http-status"; @@ -54,6 +55,10 @@ const { WALLET_EXTENSION_SECRET } = env; if (!WALLET_EXTENSION_SECRET) throw new Error("missing wallet extension secret"); const walletExtensionKey = createSecretKey(Buffer.from(WALLET_EXTENSION_SECRET, "utf8")); const auth = createAuth(database, authSecret); +const credit = { + close: vi.fn["close"]>().mockResolvedValue(), + enqueue: vi.fn["enqueue"]>(), +}; const panda = createPanda({ key: parse(pipe(string(), nonEmpty()), env.PANDA_API_KEY), url: parse(pipe(string(), nonEmpty()), env.PANDA_API_URL), @@ -65,6 +70,7 @@ const persona = createPersona( const walletExtension = createWalletExtension(WALLET_EXTENSION_SECRET); const app = route({ auth: authenticate(""), + credit, database, panda, pax: createPax({ @@ -187,6 +193,7 @@ describe("authenticated", () => { afterEach(() => vi.resetAllMocks()); beforeEach(() => { + vi.mocked(credit.enqueue).mockResolvedValue(); vi.spyOn(persona, "getAccount").mockResolvedValue(undefined); // eslint-disable-line unicorn/no-useless-undefined vi.spyOn(panda, "getCards").mockResolvedValue([]); }); @@ -658,8 +665,6 @@ describe("authenticated", () => { { id: orphanId, status: "active", last4: "4242", expirationMonth: "9", expirationYear: "2029" }, ]); const createCard = vi.spyOn(panda, "createCard"); - const error = new Error("preview failure"); - vi.spyOn(Panda, "autoCredit").mockRejectedValueOnce(error); const getAccount = vi.spyOn(persona, "getAccount"); const response = await appClient.index.$post({ header: { "test-credential-id": credentialId } }); @@ -683,7 +688,7 @@ describe("authenticated", () => { lastFour: "4242", productId: SIGNATURE_PRODUCT_ID, }); - expect(captureException).toHaveBeenCalledWith(error); + expect(credit.enqueue).toHaveBeenCalledExactlyOnceWith(padHex("0x4051", { size: 20 })); expect( vi .mocked(captureException) @@ -704,12 +709,13 @@ describe("authenticated", () => { it("adopts only the first active card when panda has multiple orphans", async () => { const credentialId = "orphan-multi"; + const account = padHex("0x4053", { size: 20 }); const first = "00000000-0000-4000-8000-0000000000c1"; const second = "00000000-0000-4000-8000-0000000000c2"; await database.insert(credentials).values({ id: credentialId, publicKey: new Uint8Array(), - account: padHex("0x4053", { size: 20 }), + account, factory: inject("ExaAccountFactory"), pandaId: credentialId, }); @@ -743,6 +749,7 @@ describe("authenticated", () => { where: eq(cards.credentialId, credentialId), }); expect(persisted).toStrictEqual([{ id: first }]); + expect(credit.enqueue).toHaveBeenCalledExactlyOnceWith(account); expect( vi .mocked(captureException) @@ -958,14 +965,6 @@ describe("authenticated", () => { vi.spyOn(panda, "createCard").mockResolvedValueOnce({ ...cardTemplate, id }); vi.spyOn(panda, "getCard").mockResolvedValueOnce({ ...cardTemplate, id }); vi.spyOn(panda, "getApplicationStatus").mockResolvedValueOnce({ id: "pandaId", applicationStatus: "approved" }); - const sigCredential = await database.query.credentials.findFirst({ - columns: { account: true }, - where: eq(credentials.id, "sig"), - }); - expect(sigCredential).toBeDefined(); - if (!sigCredential) throw new Error("missing sig credential"); - expect(await Panda.autoCredit(parse(Address, sigCredential.account))).toBe(false); - const response = await appClient.index.$post({ header: { "test-credential-id": "debit" } }); expect(response.status).toBe(200); @@ -976,6 +975,7 @@ describe("authenticated", () => { }); expect(created?.mode).toBe(0); + expect(credit.enqueue).toHaveBeenCalledExactlyOnceWith(padHex("0x4", { size: 20 })); await expect(response.json()).resolves.toStrictEqual({ status: "ACTIVE", lastFour: "7394", @@ -984,30 +984,26 @@ describe("authenticated", () => { }); }); - it("creates a panda credit card with signature product id", async () => { + it("queues credit after creating a panda card", async () => { vi.spyOn(panda, "createCard").mockResolvedValueOnce({ ...cardTemplate, id: "123e4567-e89b-12d3-a456-426655440001", last4: "1224", }); vi.spyOn(panda, "getApplicationStatus").mockResolvedValueOnce({ id: "pandaId", applicationStatus: "approved" }); - const ethCredential = await database.query.credentials.findFirst({ - columns: { account: true }, - where: eq(credentials.id, "eth"), - }); - expect(ethCredential).toBeDefined(); - if (!ethCredential) throw new Error("missing eth credential"); - const account = parse(Address, ethCredential.account); - await vi.waitUntil(async () => await Panda.autoCredit(account).catch(() => false), 26_666); - const response = await appClient.index.$post({ header: { "test-credential-id": "eth" } }); - expect(response.status).toBe(200); const created = await database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.credentialId, "eth"), }); - expect(created?.mode).toBe(1); + expect(created?.mode).toBe(0); + expect(credit.enqueue).toHaveBeenCalledExactlyOnceWith( + deriveAddress(inject("ExaAccountFactory"), { + x: padHex(privateKeyToAddress(padHex("0xbeef"))), + y: zeroHash, + }), + ); expect(captureException).not.toHaveBeenCalled(); await expect(response.json()).resolves.toStrictEqual({ status: "ACTIVE", @@ -1017,6 +1013,42 @@ describe("authenticated", () => { }); }); + it("keeps the card when credit cannot be queued", async () => { + const credentialId = crypto.randomUUID(); + const account = privateKeyToAddress(generatePrivateKey()); + const cardId = crypto.randomUUID(); + const error = new Error("queue error"); + await database.insert(credentials).values({ + id: credentialId, + publicKey: new Uint8Array(), + account, + factory: inject("ExaAccountFactory"), + pandaId: credentialId, + }); + vi.spyOn(panda, "createCard").mockResolvedValueOnce({ ...cardTemplate, id: cardId, last4: "4054" }); + vi.spyOn(panda, "getApplicationStatus").mockResolvedValueOnce({ id: "pandaId", applicationStatus: "approved" }); + vi.mocked(credit.enqueue).mockRejectedValueOnce(error); + + const response = await appClient.index.$post({ header: { "test-credential-id": credentialId } }); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toStrictEqual({ + status: "ACTIVE", + lastFour: "4054", + cardId, + productId: SIGNATURE_PRODUCT_ID, + }); + await expect( + database.query.cards.findFirst({ columns: { id: true }, where: eq(cards.credentialId, credentialId) }), + ).resolves.toStrictEqual({ id: cardId }); + expect(credit.enqueue).toHaveBeenCalledExactlyOnceWith(account); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + level: "error", + tags: { queue: "credit", job: "credit" }, + extra: { account }, + }); + }); + describe("product selection by chain", () => { const defaultChainId = chain.id; @@ -1055,7 +1087,6 @@ describe("authenticated", () => { it("issues a base product card on base", async () => { chain.id = base.id; - vi.spyOn(Panda, "autoCredit").mockResolvedValue(false); vi.spyOn(panda, "getApplicationStatus").mockResolvedValueOnce({ id: "base-default-panda", applicationStatus: "approved", @@ -1083,7 +1114,6 @@ describe("authenticated", () => { it("issues a signature product card on base for the override source", async () => { chain.id = base.id; - vi.spyOn(Panda, "autoCredit").mockResolvedValue(false); vi.spyOn(panda, "getApplicationStatus").mockResolvedValueOnce({ id: "base-signature-panda", applicationStatus: "approved", @@ -1111,7 +1141,6 @@ describe("authenticated", () => { it("issues a signature product card on optimism", async () => { chain.id = optimism.id; - vi.spyOn(Panda, "autoCredit").mockResolvedValue(false); vi.spyOn(panda, "getApplicationStatus").mockResolvedValueOnce({ id: "optimism-panda", applicationStatus: "approved", diff --git a/server/test/e2e.ts b/server/test/e2e.ts index 4738864b9f..3093c6b7ca 100644 --- a/server/test/e2e.ts +++ b/server/test/e2e.ts @@ -21,6 +21,7 @@ import createOnesignal from "../utils/onesignal"; import createPanda from "../utils/panda"; import createSardine from "../utils/sardine"; import createSegment from "../utils/segment"; +import creditWorker from "../workers/credit/worker"; import hookWorker from "../workers/hook/worker"; import refundWorker from "../workers/refund/worker"; import subscribeWorker from "../workers/subscribe/worker"; @@ -43,6 +44,7 @@ describe("e2e", () => { const sardine = createSardine("sardine", "https://sardine.test"); const segment = createSegment("segment"); const workers = [ + creditWorker({ bullmq, database, onesignal }), refundWorker({ bullmq, database, @@ -97,7 +99,6 @@ vi.mock("../utils/panda", async (importOriginal: () => Promise) => const cards = new Map(); return { ...original, - autoCredit: vi.fn().mockResolvedValue(false), signIssuerOp: vi.fn().mockResolvedValue("0x" + "ab".repeat(65)), default: (...parameters: Parameters) => ({ ...original.default(...parameters), diff --git a/server/test/hooks/activity.test.ts b/server/test/hooks/activity.test.ts index aade5edf07..9c4e3f9dc9 100644 --- a/server/test/hooks/activity.test.ts +++ b/server/test/hooks/activity.test.ts @@ -28,12 +28,11 @@ import { afterEach, beforeAll, beforeEach, describe, expect, inject, it, vi } fr import deriveAddress from "@exactly/common/deriveAddress"; import { exaAccountFactoryAbi, previewerAbi } from "@exactly/common/generated/chain"; -import database, { cards, credentials } from "../../database"; +import database, { credentials } from "../../database"; import activity from "../../hooks/activity"; import t, { f } from "../../i18n"; import createAlchemy, { activityUrl, NETWORKS, type Webhook } from "../../utils/alchemy"; import createOnesignal from "../../utils/onesignal"; -import * as Panda from "../../utils/panda"; import publicClient from "../../utils/publicClient"; import redis from "../../utils/redis"; import createSegment from "../../utils/segment"; @@ -43,6 +42,10 @@ import anvilClient from "../anvilClient"; const executor = privateKeyToAccount(padHex("0x69")); const waitForReceipt = publicClient.waitForTransactionReceipt; const alchemy = createAlchemy("webhooks"); +const credit = { + close: vi.fn<() => Promise>().mockResolvedValue(), + enqueue: vi.fn<(account: Address) => Promise>().mockResolvedValue(), +}; const onesignal = createOnesignal("onesignal"); const segment = createSegment("segment"); @@ -71,9 +74,9 @@ describe("address activity", { timeout: 66_666 }, () => { let account: Address; beforeEach(async () => { + credit.enqueue.mockReset().mockResolvedValue(); keeper = wallet(executor); vi.mocked(wallet).mockReset().mockReturnValue(keeper); - vi.spyOn(Panda, "autoCredit").mockResolvedValue(false); owner = privateKeyToAccount(generatePrivateKey()); account = deriveAddress(inject("ExaAccountFactory"), { x: padHex(owner.address), y: zeroHash }); vi.spyOn(publicClient, "waitForTransactionReceipt").mockImplementation((parameters) => @@ -567,7 +570,7 @@ describe("address activity", { timeout: 66_666 }, () => { expect(sendPushNotification).not.toHaveBeenCalled(); }); - it("pokes eth", async () => { + it("pokes eth and queues credit", async () => { const deposit = parseEther("5"); await anvilClient.setBalance({ address: account, value: deposit }); @@ -582,9 +585,11 @@ describe("address activity", { timeout: 66_666 }, () => { }, }); const market = await waitForWETHMarket(account, deposit); + await vi.waitUntil(() => credit.enqueue.mock.calls.some(([queued]) => queued === account)); expect(market.floatingDepositAssets).toBe(deposit); expect(market.isCollateral).toBe(true); + expect(credit.enqueue.mock.calls.filter(([queued]) => queued === account)).toStrictEqual([[account]]); expect(setUser).toHaveBeenCalledWith({ id: account }); expect(response.status).toBe(200); }); @@ -1047,136 +1052,6 @@ describe("address activity", { timeout: 66_666 }, () => { expect(response.status).toBe(200); }); - it("activates credit mode and sends translated notification when auto credit applies", async () => { - vi.mocked(Panda.autoCredit).mockResolvedValue(true); - const { sendPushNotification, sent } = mockCredit(); - await database.insert(cards).values([{ id: "auto-credit", credentialId: account, lastFour: "1234", mode: 0 }]); - await anvilClient.writeContract({ - account: null, - address: inject("WETH"), - abi: mockERC20Abi, - functionName: "mint", - args: [account, parseEther(String(activityPayload.json.event.activity[1].value))], - }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [ - { - ...activityPayload.json.event.activity[1], - toAddress: account, - rawContract: { ...activityPayload.json.event.activity[1].rawContract, address: inject("WETH") }, - }, - ], - }, - }, - }); - - await sent; - expect(sendPushNotification).toHaveBeenCalledWith({ - userId: account, - headings: t("Card mode changed"), - contents: t("Credit mode activated"), - }); - expect(vi.mocked(captureException).mock.calls.some(([error, hint]) => isNoBalance(error, hint, "warning"))).toBe( - false, - ); - expect(response.status).toBe(200); - }); - - it("captures auto credit notification errors", async () => { - const error = new Error("push failed"); - vi.mocked(Panda.autoCredit).mockResolvedValue(true); - const notification = Promise.withResolvers>>(); - const { sendPushNotification, sent } = mockCredit(notification.promise); - await database - .insert(cards) - .values([{ id: "auto-credit-notify-error", credentialId: account, lastFour: "8765", mode: 0 }]); - await anvilClient.writeContract({ - account: null, - address: inject("WETH"), - abi: mockERC20Abi, - functionName: "mint", - args: [account, parseEther(String(activityPayload.json.event.activity[1].value))], - }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [ - { - ...activityPayload.json.event.activity[1], - toAddress: account, - rawContract: { ...activityPayload.json.event.activity[1].rawContract, address: inject("WETH") }, - }, - ], - }, - }, - }); - - await sent; - expect(sendPushNotification).toHaveBeenCalledWith({ - userId: account, - headings: t("Card mode changed"), - contents: t("Credit mode activated"), - }); - notification.reject(error); - await notification.promise.catch(() => undefined); - expect(captureException).toHaveBeenCalledWith(error); - expect( - vi.mocked(captureException).mock.calls.some(([captured, hint]) => isNoBalance(captured, hint, "warning")), - ).toBe(false); - expect(response.status).toBe(200); - }); - - it("captures auto credit errors", async () => { - const error = new Error("auto credit"); - const autoCredit = vi.mocked(Panda.autoCredit).mockRejectedValue(error); - await database - .insert(cards) - .values([{ id: "auto-credit-error", credentialId: account, lastFour: "4321", mode: 0 }]); - await anvilClient.writeContract({ - account: null, - address: inject("WETH"), - abi: mockERC20Abi, - functionName: "mint", - args: [account, parseEther(String(activityPayload.json.event.activity[1].value))], - }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [ - { - ...activityPayload.json.event.activity[1], - toAddress: account, - rawContract: { ...activityPayload.json.event.activity[1].rawContract, address: inject("WETH") }, - }, - ], - }, - }, - }); - - await waitForActivity(); - - expect(autoCredit).toHaveBeenCalledWith(account); - expect(captureException).toHaveBeenCalledWith(error); - expect( - vi.mocked(captureException).mock.calls.some(([captured, hint]) => isNoBalance(captured, hint, "warning")), - ).toBe(false); - expect(response.status).toBe(200); - }); - it("doesn't send a notification for market shares", async () => { const sendPushNotification = sendPushNotificationMock; @@ -1586,22 +1461,6 @@ function mockLifiTokens(response: Error | Record }); } -function mockCredit(response?: ReturnType) { - const sent = Promise.withResolvers(); - const send = sendPushNotificationMock.getMockImplementation(); - if (!send) throw new Error("missing onesignal mock"); - const sendPushNotification = sendPushNotificationMock.mockImplementation((input) => { - if ( - JSON.stringify(input.headings) !== JSON.stringify(t("Card mode changed")) || - JSON.stringify(input.contents) !== JSON.stringify(t("Credit mode activated")) - ) - return send(input); - sent.resolve(true); - return response ?? send(input); - }); - return { sendPushNotification, sent: sent.promise }; -} - const mockERC20Abi = [ { type: "function", @@ -1684,6 +1543,7 @@ afterEach(async () => { function createHook(current = alchemy) { return activity({ alchemy: current, + credit, database, executor, onesignal, diff --git a/server/test/workers/bin.test.ts b/server/test/workers/bin.test.ts index 5f31f546f4..b05593ca74 100644 --- a/server/test/workers/bin.test.ts +++ b/server/test/workers/bin.test.ts @@ -16,6 +16,7 @@ const mocks = { alchemy: vi.fn<(key: string) => object>(), chat: vi.fn<(config: { anthropicKey: string; bullmq: object; whatsapp: object }) => Handle>(), close: vi.fn<() => Promise>(), + credit: vi.fn<(config: { bullmq: object; database: typeof database; onesignal: object }) => Handle>(), drizzle: vi.fn<() => typeof database>(), hook: vi.fn<(config: { bullmq: object; database: typeof database; panda: object }) => Handle>(), onesignal: vi.fn<(key: string) => object>(), @@ -49,6 +50,7 @@ beforeEach(() => { vi.resetModules(); mocks.alchemy.mockReset().mockReturnValue(alchemy); mocks.close.mockReset().mockResolvedValue(); + mocks.credit.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.drizzle.mockReset().mockReturnValue(database); mocks.hook.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.onesignal.mockReset().mockReturnValue(onesignal); @@ -84,6 +86,7 @@ beforeEach(() => { vi.doMock("../../utils/wallet", () => ({ signer: mocks.signer })); vi.doMock("../../utils/whatsapp", () => ({ default: mocks.whatsapp })); vi.doMock("../../workers/chat/worker", () => ({ default: mocks.chat })); + vi.doMock("../../workers/credit/worker", () => ({ default: mocks.credit })); vi.doMock("../../workers/hook/worker", () => ({ default: mocks.hook })); vi.doMock("../../workers/refund/worker", () => ({ default: mocks.refund })); vi.doMock("../../workers/subscribe/worker", () => ({ default: mocks.subscribe })); @@ -114,6 +117,27 @@ describe("bin", () => { }); }); + it("resolves credit private config before constructing and supervising its worker", async () => { + await import("../../workers/credit/bin"); + + const created = mocks.supervise.mock.calls[0]?.[1]; + if (!created) throw new Error("missing worker"); + expect(mocks.supervise).toHaveBeenCalledExactlyOnceWith("credit", created); + await created; + expect(mocks.secret.mock.calls.map(([secret]) => secret)).toStrictEqual([ + "redis-url", + "credit-postgres-url", + "credit-onesignal-api-key", + ]); + expect(new Set(mocks.secret.mock.calls.map(([, secrets]) => secrets)).size).toBe(1); + expect(mocks.onesignal).toHaveBeenCalledExactlyOnceWith("credit-onesignal-api-key"); + expect(mocks.credit).toHaveBeenCalledExactlyOnceWith({ + bullmq: expect.objectContaining({ redisUrl: "redis-url", options: { maxRetriesPerRequest: null } }) as object, + database, + onesignal, + }); + }); + it("resolves refund private config before constructing and supervising its worker", async () => { await import("../../workers/refund/bin"); diff --git a/server/test/workers/credit.test.ts b/server/test/workers/credit.test.ts new file mode 100644 index 0000000000..39987cf3e5 --- /dev/null +++ b/server/test/workers/credit.test.ts @@ -0,0 +1,351 @@ +import sendPushNotificationMock from "../mocks/onesignal"; +import "../mocks/sentry"; + +import { captureException, continueTrace, startSpan, withScope } from "@sentry/node"; +import { Queue, QueueEvents } from "bullmq"; +import { eq } from "drizzle-orm"; +import { parse } from "valibot"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +import { marketUSDCAddress } from "@exactly/common/generated/chain"; +import { Address } from "@exactly/common/validation"; + +import database, { cards, credentials } from "../../database"; +import t from "../../i18n"; +import createOnesignal from "../../utils/onesignal"; +import publicClient from "../../utils/publicClient"; +import { bullmq } from "../../utils/redis"; +import createCredit from "../../workers/credit/queue"; +import creditWorker from "../../workers/credit/worker"; + +import type { Job as Credit } from "../../workers/credit/job"; +import type * as sentry from "@sentry/node"; +import type { JobsOptions } from "bullmq"; + +const account = parse(Address, "0xb12057309bdDd6e071d5AAF9714C5f15E02441D6"); +const unknown = parse(Address, "0x1234567890123456789012345678901234567890"); +const market = parse(Address, "0xafc70edeb980d345da3c76786d9689d41804b521"); +const credit = createCredit(bullmq); +const onesignal = createOnesignal("onesignal"); +const queue = new Queue("credit", { connection: bullmq }); +const events = new QueueEvents("credit", { connection: bullmq }); +let worker: ReturnType; + +beforeAll(async () => { + await database.insert(credentials).values({ + id: "credit-worker", + account, + factory: parse(Address, "0x9876543210987654321098765432109876543210"), + publicKey: new Uint8Array(), + }); + await database.insert(cards).values({ id: "credit-card", credentialId: "credit-worker", lastFour: "1234" }); +}); + +beforeEach(async () => { + vi.restoreAllMocks(); + sendPushNotificationMock.mockResolvedValue({}); + vi.spyOn(publicClient, "readContract").mockResolvedValue([] as never); + vi.clearAllMocks(); + await database.update(cards).set({ mode: 0, status: "ACTIVE" }).where(eq(cards.id, "credit-card")); + await queue.drain(true); + await queue.clean(0, 1000, "completed"); + await queue.clean(0, 1000, "failed"); +}); + +afterAll(async () => { + await database.delete(cards).where(eq(cards.credentialId, "credit-worker")); + await database.delete(credentials).where(eq(credentials.id, "credit-worker")); + await Promise.all([queue.close(), events.close(), credit.close()]); +}); + +describe("credit queue", () => { + it("publishes automatic credit jobs", async () => { + await expect(credit.enqueue(account)).resolves.toBeUndefined(); + + const job = await queue.getJob(account); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(account); + expect(job.name).toBe("credit"); + expect(job.data).toStrictEqual({ + account, + sentryBaggage: expect.any(String) as string, + sentryTrace: expect.any(String) as string, + }); + expect(job.opts).toStrictEqual({ + attempts: 10, + backoff: { type: "exponential", delay: 1000 }, + jobId: account, + removeOnComplete: true, + removeOnFail: { count: 1000, age: 7 * 24 * 3600 }, + }); + await expect(job.getState()).resolves.toBe("waiting"); + expect(startSpan).toHaveBeenCalledWith( + expect.objectContaining({ name: "credit", op: "queue.publish" }), + expect.any(Function), + ); + expect(captureException).not.toHaveBeenCalled(); + await job.remove(); + }); + + it("propagates queue failures", async () => { + const error = new Error("queue error"); + vi.spyOn(Queue.prototype, "add").mockRejectedValueOnce(error); + + await expect(credit.enqueue(account)).rejects.toThrow(error); + + expect(captureException).not.toHaveBeenCalled(); + }); +}); + +describe("credit worker", () => { + beforeAll(async () => { + worker = creditWorker({ bullmq, database, onesignal }); + await worker.ready; + }); + + afterAll(async () => { + await worker.close(); + await worker.close(); + }); + + it("automatically activates credit mode", async () => { + const setAttribute = await spySpanSetAttribute(); + vi.mocked(publicClient.readContract).mockResolvedValueOnce([{ floatingDepositAssets: 1n, market }] as never); + + await jobFinished(account); + + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 1 }); + expect(sendPushNotificationMock).toHaveBeenCalledExactlyOnceWith({ + userId: account, + headings: t("Credit mode activated"), + contents: t("Your card is now in credit mode"), + }); + expect(setAttribute.mock.calls.filter(([attribute]) => String(attribute).startsWith("exa."))).toStrictEqual([ + ["exa.autoCredit", true], + ["exa.card", "credit-card"], + ["exa.mode", 1], + ]); + expect(startSpan).toHaveBeenCalledWith( + expect.objectContaining({ forceTransaction: true, name: "credit worker" }), + expect.any(Function), + ); + expect(startSpan).toHaveBeenCalledWith( + expect.objectContaining({ name: "credit", op: "queue.process" }), + expect.any(Function), + ); + expect(captureException).not.toHaveBeenCalled(); + }); + + it("keeps debit mode without deposits", async () => { + await jobFinished(account); + + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 0 }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + }); + + it("ignores empty deposits", async () => { + vi.mocked(publicClient.readContract).mockResolvedValueOnce([{ floatingDepositAssets: 0n, market }] as never); + + await jobFinished(account); + + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 0 }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + }); + + it("keeps debit mode with usdc deposits", async () => { + vi.mocked(publicClient.readContract).mockResolvedValueOnce([ + { floatingDepositAssets: 1n, market }, + { floatingDepositAssets: 1n, market: marketUSDCAddress }, + ] as never); + + await jobFinished(account); + + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 0 }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + }); + + it("does not change an existing credit mode", async () => { + await database.update(cards).set({ mode: 1 }).where(eq(cards.id, "credit-card")); + vi.mocked(publicClient.readContract).mockResolvedValueOnce([{ floatingDepositAssets: 1n, market }] as never); + + await jobFinished(account); + + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 1 }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + }); + + it("does not change deleted cards", async () => { + await database.update(cards).set({ status: "DELETED" }).where(eq(cards.id, "credit-card")); + vi.mocked(publicClient.readContract).mockResolvedValueOnce([{ floatingDepositAssets: 1n, market }] as never); + + await jobFinished(account); + + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 0 }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + }); + + it("does not change cards for unknown accounts", async () => { + vi.mocked(publicClient.readContract).mockResolvedValueOnce([{ floatingDepositAssets: 1n, market }] as never); + + await jobFinished(unknown); + + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 0 }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + }); + + it("captures notification errors without retrying", async () => { + const error = new Error("push failed"); + vi.mocked(publicClient.readContract).mockResolvedValueOnce([{ floatingDepositAssets: 1n, market }] as never); + sendPushNotificationMock.mockRejectedValueOnce(error); + + await jobFinished(account); + + expect(captureException).toHaveBeenCalledExactlyOnceWith(error); + expect(publicClient.readContract).toHaveBeenCalledOnce(); + await expect( + database.query.cards.findFirst({ columns: { mode: true }, where: eq(cards.id, "credit-card") }), + ).resolves.toStrictEqual({ mode: 1 }); + }); + + it("retries automatic credit failures", async () => { + vi.mocked(publicClient.readContract) + .mockRejectedValueOnce(new Error("rpc unavailable")) + .mockResolvedValueOnce([{ floatingDepositAssets: 1n, market }] as never); + + await jobFinished(account, { attempts: 2, backoff: { type: "fixed", delay: 1 } }); + + expect(publicClient.readContract).toHaveBeenCalledTimes(2); + expect(captureException).not.toHaveBeenCalled(); + }); + + it("captures persisted terminal failures", async () => { + const error = new Error("credit failed"); + vi.mocked(publicClient.readContract).mockRejectedValueOnce(error); + const setUser = await spyScopeSetUser(); + + await expect(jobFinished(account, { removeOnFail: false })).rejects.toThrow("credit failed"); + + await expect(queue.getFailedCount()).resolves.toBe(1); + await expect(queue.getJobState(account)).resolves.toBe("failed"); + const [job] = await queue.getFailed(); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(account); + expect(job.failedReason).toBe("credit failed"); + expect(job.attemptsMade).toBe(1); + expect(job.stacktrace).toHaveLength(1); + expect(setUser).toHaveBeenCalledExactlyOnceWith({ id: account }); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account, attempts: 1, id: account }, + level: "error", + tags: { queue: "credit", job: "credit" }, + }); + await job.remove(); + }); + + it("captures only terminal failed events", async () => { + const error = new Error("credit failed"); + vi.mocked(publicClient.readContract).mockRejectedValue(error); + const setUser = await spyScopeSetUser(); + + await expect(jobFinished(account, { attempts: 2, backoff: { type: "fixed", delay: 1 } })).rejects.toThrow( + "credit failed", + ); + + expect(publicClient.readContract).toHaveBeenCalledTimes(2); + expect(setUser).toHaveBeenCalledExactlyOnceWith({ id: account }); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account, attempts: 2, id: account }, + level: "error", + tags: { queue: "credit", job: "credit" }, + }); + }); + + it("continues sentry traces", async () => { + await jobFinished(account, undefined, { sentryBaggage: "baggage", sentryTrace: "trace" }); + + expect(continueTrace).toHaveBeenCalledWith({ sentryTrace: "trace", baggage: "baggage" }, expect.any(Function)); + }); + + it("captures worker errors", () => { + const error = new Error("worker error"); + + worker.queue.emit("error", error); + + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { level: "error", tags: { queue: "credit" } }); + }); + + it("captures failed events without a job", async () => { + const error = new Error("failed event error"); + const setUser = await spyScopeSetUser(); + + worker.queue.emit("failed", undefined, error, "active"); + + expect(setUser).not.toHaveBeenCalled(); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account: undefined, attempts: undefined, id: undefined }, + level: "error", + tags: { queue: "credit", job: undefined }, + }); + }); +}); + +async function jobFinished( + current: Address, + options?: JobsOptions, + trace?: Pick, +) { + const job = await queue.add( + "credit", + { account: current, ...trace }, + { attempts: 1, jobId: current, removeOnComplete: true, removeOnFail: true, ...options }, + ); + await job.waitUntilFinished(events).catch(async (error: unknown) => { + await vi.waitUntil(() => vi.mocked(captureException).mock.calls.length > 0); + throw error; + }); +} + +async function spyScopeSetUser() { + const { withScope: realWithScope } = await vi.importActual("@sentry/node"); + const setUser = vi.fn(); + vi.mocked(withScope).mockImplementation((_scopeOrCallback, _callback?) => + realWithScope((scope) => { + const originalSetUser = scope.setUser.bind(scope); + scope.setUser = (...args: Parameters) => { + setUser(...args); + return originalSetUser(...args); + }; + return ((_callback ?? _scopeOrCallback) as NonNullable)(scope); + }), + ); + return setUser; +} + +async function spySpanSetAttribute() { + const { startSpan: realStartSpan } = await vi.importActual("@sentry/node"); + const setAttribute = vi.fn(); + vi.mocked(startSpan).mockImplementation(((options, callback) => + realStartSpan(options, (span) => { + const originalSetAttribute = span.setAttribute.bind(span); + span.setAttribute = (...args: Parameters) => { + setAttribute(...args); + return originalSetAttribute(...args); + }; + return callback(span); + })) as typeof startSpan); + return setAttribute; +} diff --git a/server/utils/panda.ts b/server/utils/panda.ts index e23eafa7c2..e34a370abb 100644 --- a/server/utils/panda.ts +++ b/server/utils/panda.ts @@ -38,18 +38,11 @@ import { import { recoverTypedDataAddress, type LocalAccount } from "viem"; import { base, baseSepolia, optimism, optimismSepolia } from "viem/chains"; -import chain, { - issuerCheckerAddress, - marketUSDCAddress, - previewerAbi, - previewerAddress, - usdcAddress, -} from "@exactly/common/generated/chain"; +import chain, { issuerCheckerAddress, usdcAddress } from "@exactly/common/generated/chain"; import { BASE_PRODUCT_ID, PLATINUM_PRODUCT_ID, SIGNATURE_PRODUCT_ID } from "@exactly/common/panda"; import { Address, Hex } from "@exactly/common/validation"; import { proposalManager } from "@exactly/plugin/deploy.json"; -import publicClient from "./publicClient"; import ServiceError from "./ServiceError"; import verifySignature from "./verifySignature"; @@ -343,23 +336,6 @@ export default function panda({ key, url }: { key: string; url: string }) { } } -export async function autoCredit(account: Address) { - const markets = await publicClient.readContract({ - address: previewerAddress, - functionName: "exactly", - abi: previewerAbi, - args: [account], - }); - let hasCollateral = false; - for (const { floatingDepositAssets, market } of markets) { - if (floatingDepositAssets > 0n) { - if (market === marketUSDCAddress) return false; - hasCollateral = true; - } - } - return hasCollateral; -} - async function verifyPandaSignature( { account, diff --git a/server/workers/credit/bin.ts b/server/workers/credit/bin.ts new file mode 100644 index 0000000000..99bf541310 --- /dev/null +++ b/server/workers/credit/bin.ts @@ -0,0 +1,28 @@ +import { SecretManagerServiceClient } from "@google-cloud/secret-manager"; +import { drizzle } from "drizzle-orm/node-postgres"; + +import { name } from "./job"; +import worker from "./worker"; +import * as schema from "../../database/schema"; +import supervise, { own } from "../../supervise"; +import createOnesignal from "../../utils/onesignal"; +import secret from "../../utils/secret"; +import { connect } from "../worker"; + +const secrets = new SecretManagerServiceClient(); + +supervise( + name, + Promise.all([ + secret("redis-url", secrets).then((url) => connect(url)), + secret("credit-postgres-url", secrets).then((url) => drizzle(url, { schema })), + secret("credit-onesignal-api-key", secrets).then((key) => createOnesignal(key)), + ]).then(([bullmq, database, onesignal]) => + own( + worker({ bullmq, database, onesignal }), + () => bullmq.quit(), + () => database.$client.end(), + () => secrets.close(), + ), + ), +); diff --git a/server/workers/credit/job.ts b/server/workers/credit/job.ts new file mode 100644 index 0000000000..ec63bae769 --- /dev/null +++ b/server/workers/credit/job.ts @@ -0,0 +1,10 @@ +import type { Address } from "@exactly/common/validation"; + +export const name = "credit"; +export const attempts = 10; + +export type Job = { + account: Address; + sentryBaggage?: string; + sentryTrace?: string; +}; diff --git a/server/workers/credit/queue.ts b/server/workers/credit/queue.ts new file mode 100644 index 0000000000..45a03933bd --- /dev/null +++ b/server/workers/credit/queue.ts @@ -0,0 +1,14 @@ +import { attempts, name, type Job } from "./job"; +import createQueue from "../queue"; + +import type { Redis } from "ioredis"; + +export default function queue(redis: Redis) { + const instance = createQueue(name, attempts, redis); + return { + close: () => instance.close(), + async enqueue(account: Job["account"]) { + await instance.enqueue({ account }, account); + }, + }; +} diff --git a/server/workers/credit/worker.ts b/server/workers/credit/worker.ts new file mode 100644 index 0000000000..b2ee8798fd --- /dev/null +++ b/server/workers/credit/worker.ts @@ -0,0 +1,84 @@ +import { captureException, withScope } from "@sentry/node"; +import { eq, inArray } from "drizzle-orm"; + +import { marketUSDCAddress, previewerAbi, previewerAddress } from "@exactly/common/generated/chain"; + +import { attempts, name, type Job } from "./job"; +import { cards, credentials } from "../../database/schema"; +import t from "../../i18n"; +import publicClient from "../../utils/publicClient"; +import createWorker from "../worker"; + +import type * as schema from "../../database/schema"; +import type createOnesignal from "../../utils/onesignal"; +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import type { Redis } from "ioredis"; + +export default function worker({ + bullmq, + database, + onesignal, +}: { + bullmq: Redis; + database: NodePgDatabase; + onesignal: ReturnType; +}) { + return createWorker({ + attempts, + bullmq, + failed(job, error) { + withScope((scope) => { + if (job) scope.setUser({ id: job.data.account }); + captureException(error, { + extra: { account: job?.data.account, attempts: job?.attemptsMade, id: job?.id }, + level: "error", + tags: { queue: name, job: job?.name }, + }); + }); + }, + name, + async process(job, span) { + const markets = await publicClient.readContract({ + address: previewerAddress, + functionName: "exactly", + abi: previewerAbi, + args: [job.data.account], + }); + let auto = false; + for (const { floatingDepositAssets, market } of markets) { + if (floatingDepositAssets <= 0n) continue; + if (market === marketUSDCAddress) { + auto = false; + break; + } + auto = true; + } + span.setAttribute("exa.autoCredit", auto); + if (auto) { + const credential = await database.query.credentials.findFirst({ + where: eq(credentials.account, job.data.account), + columns: {}, + with: { + cards: { + columns: { id: true, mode: true }, + where: inArray(cards.status, ["ACTIVE", "FROZEN"]), + }, + }, + }); + const card = credential?.cards[0]; + span.setAttribute("exa.card", card?.id); + if (card?.mode === 0) { + await database.update(cards).set({ mode: 1 }).where(eq(cards.id, card.id)); + span.setAttribute("exa.mode", 1); + await onesignal + .sendPushNotification({ + userId: job.data.account, + headings: t("Credit mode activated"), + contents: t("Your card is now in credit mode"), + }) + .catch((error: unknown) => captureException(error)); + } + } + }, + }); +} From 30b501646741a4e91be07cb8140a58f70138682f Mon Sep 17 00:00:00 2001 From: danilo neves cruz Date: Wed, 22 Jul 2026 17:01:04 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=A8=20server:=20process=20poke=20in?= =?UTF-8?q?=20worker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit co-authored-by: Miguel Diaz --- .changeset/bumpy-regions-read.md | 5 + .changeset/proud-tools-sneeze.md | 5 + infra/Pulumi.base.yaml | 1 + infra/Pulumi.production.yaml | 1 + infra/utils/modules.ts | 1 + server/hooks/activity.ts | 159 +---- server/i18n/es.json | 2 + server/i18n/pt.json | 2 + server/index.ts | 7 +- server/test/e2e.ts | 2 + server/test/hooks/activity.test.ts | 996 ++++------------------------- server/test/workers/bin.test.ts | 48 +- server/test/workers/poke.test.ts | 706 ++++++++++++++++++++ server/workers/credit/queue.ts | 9 +- server/workers/poke/bin.ts | 32 + server/workers/poke/job.ts | 16 + server/workers/poke/queue.ts | 25 + server/workers/poke/worker.ts | 174 +++++ 18 files changed, 1178 insertions(+), 1013 deletions(-) create mode 100644 .changeset/bumpy-regions-read.md create mode 100644 .changeset/proud-tools-sneeze.md create mode 100644 server/test/workers/poke.test.ts create mode 100644 server/workers/poke/bin.ts create mode 100644 server/workers/poke/job.ts create mode 100644 server/workers/poke/queue.ts create mode 100644 server/workers/poke/worker.ts diff --git a/.changeset/bumpy-regions-read.md b/.changeset/bumpy-regions-read.md new file mode 100644 index 0000000000..68c9c2d37b --- /dev/null +++ b/.changeset/bumpy-regions-read.md @@ -0,0 +1,5 @@ +--- +"@exactly/infra": minor +--- + +🧱 setup poke queue worker diff --git a/.changeset/proud-tools-sneeze.md b/.changeset/proud-tools-sneeze.md new file mode 100644 index 0000000000..b21b32e5d7 --- /dev/null +++ b/.changeset/proud-tools-sneeze.md @@ -0,0 +1,5 @@ +--- +"@exactly/server": patch +--- + +✨ process poke in worker diff --git a/infra/Pulumi.base.yaml b/infra/Pulumi.base.yaml index 888e5be6d8..66a02b699e 100644 --- a/infra/Pulumi.base.yaml +++ b/infra/Pulumi.base.yaml @@ -1,4 +1,5 @@ encryptionsalt: v1::v1:AAAAAAAAAAAAAAAb:rYjVO7/uF2+Qv1LhWq5c2S0ZR4rZaQ== config: gcp:project: eexxxaa + exa:pokeMinimum: 1 exa:subscribeTimeout: 900s diff --git a/infra/Pulumi.production.yaml b/infra/Pulumi.production.yaml index 2948181c47..8b67fad4b9 100644 --- a/infra/Pulumi.production.yaml +++ b/infra/Pulumi.production.yaml @@ -2,5 +2,6 @@ encryptionsalt: v1::v1:AAAAAAAAAAAAAAAb:rYjVO7/uF2+Qv1LhWq5c2S0ZR4rZaQ== config: gcp:project: eexxxaa exa:domain: web.exactly.app + exa:pokeMinimum: 1 exa:subscribeTimeout: 900s exa:whatsappPhoneNumberId: "1287864854409817" diff --git a/infra/utils/modules.ts b/infra/utils/modules.ts index 7fab61989e..bd996dcf7a 100644 --- a/infra/utils/modules.ts +++ b/infra/utils/modules.ts @@ -14,6 +14,7 @@ export default define({ }, credit: { secrets: ["onesignal-api-key", "postgres-url"] }, hook: { secrets: ["panda-api-key", "postgres-url"], shared: ["panda-api-url"] }, + poke: { secrets: ["onesignal-api-key", "segment-write-key"], signers: ["poker"] }, refund: { secrets: ["panda-api-key", "onesignal-api-key", "postgres-url", "sardine-api-key", "segment-write-key"], shared: ["panda-api-url", "sardine-api-url"], diff --git a/server/hooks/activity.ts b/server/hooks/activity.ts index bd57b5be86..a24921b45e 100644 --- a/server/hooks/activity.ts +++ b/server/hooks/activity.ts @@ -1,52 +1,34 @@ import { vValidator } from "@hono/valibot-validator"; -import { SPAN_STATUS_ERROR, SPAN_STATUS_OK } from "@sentry/core"; import { captureException, - continueTrace, getActiveSpan, - getTraceData, SEMANTIC_ATTRIBUTE_SENTRY_OP, setContext, setTag, setUser, - startSpan, - withScope, } from "@sentry/node"; import createDebug from "debug"; import { inArray } from "drizzle-orm"; import { Hono } from "hono"; import { validator } from "hono/validator"; import * as v from "valibot"; -import { bytesToBigInt, hexToBigInt, withRetry, type LocalAccount } from "viem"; +import { bytesToHex, hexToBigInt } from "viem"; import { anvil } from "viem/chains"; -import exaChain, { - auditorAbi, - exaAccountFactoryAbi, - exaPluginAbi, - exaPreviewerAbi, - exaPreviewerAddress, - marketAbi, - upgradeableModularAccountAbi, - wethAddress, -} from "@exactly/common/generated/chain"; +import exaChain, { exaPreviewerAbi, exaPreviewerAddress, wethAddress } from "@exactly/common/generated/chain"; import { Address, Hash, Hex } from "@exactly/common/validation"; import { credentials } from "../database/schema"; import t, { f } from "../i18n"; import { activityNetworks, activityUrl, NETWORKS } from "../utils/alchemy"; -import decodePublicKey from "../utils/decodePublicKey"; import publicClient from "../utils/publicClient"; -import revertFingerprint from "../utils/revertFingerprint"; import validatorHook from "../utils/validatorHook"; import verifySignature from "../utils/verifySignature"; -import createWallet from "../utils/wallet"; import type * as schema from "../database/schema"; import type createAlchemy from "../utils/alchemy"; import type createOnesignal from "../utils/onesignal"; -import type createSegment from "../utils/segment"; -import type createCredit from "../workers/credit/queue"; +import type createPoke from "../workers/poke/queue"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { Redis } from "ioredis"; @@ -58,20 +40,16 @@ Object.assign(debug, { inspectOpts: { depth: undefined } }); export default function hook({ alchemy, - credit, database, - executor, onesignal, + poke, redis, - segment, }: { alchemy: ReturnType; - credit: ReturnType; database: NodePgDatabase; - executor: LocalAccount; onesignal: ReturnType; + poke: ReturnType; redis: Redis; - segment: ReturnType; }) { const networks = activityNetworks(); let entries = new Map(); @@ -206,124 +184,19 @@ export default function hook({ pokes.set(account, { publicKey, factory, source, assets: new Set([asset]) }); } } - const { "sentry-trace": sentryTrace, baggage } = getTraceData(); - const wallet = createWallet(executor, chain); - Promise.allSettled( - [...pokes].map(([account, { publicKey, factory, source, assets }]) => - continueTrace({ sentryTrace, baggage }, () => - withScope((scope) => - startSpan( - { name: "account activity", op: "exa.activity", attributes: { account }, forceTransaction: true }, - async (span) => { - scope.setUser({ id: account }); - const isDeployed = !!(await wallet.getCode({ address: account })); - scope.setTag("exa.new", !isDeployed); - if (!isDeployed) { - try { - await wallet.exaSend( - { name: "create account", op: "exa.account", attributes: { account } }, - { - address: factory, - functionName: "createAccount", - args: [0n, [decodePublicKey(publicKey, bytesToBigInt)]], - abi: exaAccountFactoryAbi, - }, - chain.id === exaChain.id ? undefined : { fees: "auto" }, - ); - segment.track({ event: "AccountFunded", userId: account, properties: { source } }); - } catch (error: unknown) { - span.setStatus({ code: SPAN_STATUS_ERROR, message: "account_failed" }); - throw error; - } - } - if (chain.id !== exaChain.id) { - span.setStatus({ code: SPAN_STATUS_OK }); - return; - } - if (assets.has(ETH)) assets.delete(WETH); - const results = await Promise.allSettled( - [...assets] - .filter((asset) => marketsByAsset.has(asset) || asset === ETH) - .map(async (asset) => - withRetry( - () => - wallet - .exaSend( - { name: "poke account", op: "exa.poke", attributes: { account, asset } }, - { - address: account, - abi: [...exaPluginAbi, ...upgradeableModularAccountAbi, ...auditorAbi, ...marketAbi], - ...(asset === ETH - ? { functionName: "pokeETH" } - : { - functionName: "poke", - args: [marketsByAsset.get(asset)!], // eslint-disable-line @typescript-eslint/no-non-null-assertion - }), - }, - { ignore: ["NoBalance()"] }, - ) - .then((receipt) => { - if (receipt) return receipt; - throw new Error("NoBalance()"); - }), - { - delay: 2000, - retryCount: 5, - shouldRetry: ({ error }) => { - if (error instanceof Error && error.message === "NoBalance()") return true; - withScope((captureScope) => { - captureScope.setUser({ id: account }); - captureException(error, { level: "error", fingerprint: revertFingerprint(error) }); - }); - return true; - }, - }, - ), - ), - ); - for (const result of results) { - if (result.status === "fulfilled") { - await credit.enqueue(account); - continue; - } - if (result.reason instanceof Error && result.reason.message === "NoBalance()") { - withScope((captureScope) => { - captureScope.setUser({ id: account }); - captureScope.addEventProcessor((event) => { - if (event.exception?.values?.[0]) event.exception.values[0].type = "NoBalance"; - return event; - }); - captureException(result.reason, { - level: "warning", - fingerprint: ["{{ default }}", "NoBalance"], - }); - }); - continue; - } - span.setStatus({ code: SPAN_STATUS_ERROR, message: "poke_failed" }); - throw result.reason; - } - span.setStatus({ code: SPAN_STATUS_OK }); - }, - ), - ), - ).catch((error: unknown) => { - withScope((scope) => { - scope.setUser({ id: account }); - captureException(error, { level: "error", fingerprint: revertFingerprint(error) }); - }); - throw error; + await Promise.all( + [...pokes].map(([account, { assets, factory, publicKey, source }]) => + poke.enqueue({ + account, + assets: [...assets], + chainId: chain.id, + factory, + origin: "activity", + publicKey: bytesToHex(publicKey), + source, }), ), - ) - .then((results) => { - getActiveSpan()?.setStatus( - results.every((result) => result.status === "fulfilled") - ? { code: SPAN_STATUS_OK } - : { code: SPAN_STATUS_ERROR, message: "activity_failed" }, - ); - }) - .catch((error: unknown) => captureException(error)); + ); return c.json({}); }, ); diff --git a/server/i18n/es.json b/server/i18n/es.json index 74e08c5e11..ed6c3b852b 100644 --- a/server/i18n/es.json +++ b/server/i18n/es.json @@ -2,6 +2,7 @@ "atm limit reached. maximum 250 usd per transaction.": "límite de cajero alcanzado. máximo 250 usd por transacción.", "advertising limit reached": "se ha alcanzado el límite de compras de publicidad", "atm limit reached": "se ha alcanzado el límite de retiros en cajero", + "Account assets updated": "Activos de cuenta actualizados", "Card purchase": "Compra con tarjeta", "Credit mode activated": "Modo crédito activado", "Deposited funds": "Fondos depositados", @@ -18,6 +19,7 @@ "Your debt is due in 24 hours. Repay now to avoid penalties.": "Tu deuda vence en 24 horas. Repágala ahora para evitar penalidades.", "Your fiat onramp account has been activated": "Tu cuenta para depositar dinero fiat ha sido activada", "Your funds are on the way to your bank": "Tus fondos están en camino a tu banco", + "Your funds are ready to use": "Tus fondos están listos para usar", "card canceled": "tarjeta cancelada", "card not active": "tarjeta inactiva", "card limit exceeded": "límite de la tarjeta excedido", diff --git a/server/i18n/pt.json b/server/i18n/pt.json index 750b9ccb90..e98746d435 100644 --- a/server/i18n/pt.json +++ b/server/i18n/pt.json @@ -2,6 +2,7 @@ "atm limit reached. maximum 250 usd per transaction.": "limite de saque atingido. máximo de 250 usd por transação.", "advertising limit reached": "limite de publicidade atingido", "atm limit reached": "limite de saque atingido", + "Account assets updated": "Ativos da conta atualizados", "Card purchase": "Compra no cartão", "Credit mode activated": "Modo crédito ativado", "Deposited funds": "Fundos depositados", @@ -18,6 +19,7 @@ "Your debt is due in 24 hours. Repay now to avoid penalties.": "Sua dívida vence em 24 horas. Pague agora para evitar penalidades.", "Your fiat onramp account has been activated": "Sua conta para depositar dinheiro fiat foi ativada", "Your funds are on the way to your bank": "Seus fundos estão a caminho do seu banco", + "Your funds are ready to use": "Seus fundos estão prontos para usar", "card canceled": "cartão cancelado", "card not active": "cartão inativo", "card limit exceeded": "limite do cartão excedido", diff --git a/server/index.ts b/server/index.ts index a7c40dca00..a5becd8ba8 100644 --- a/server/index.ts +++ b/server/index.ts @@ -39,6 +39,7 @@ import createWalletExtension from "./utils/walletExtension"; import createCredit from "./workers/credit/queue"; import createHook from "./workers/hook/queue"; import createHookWorker from "./workers/hook/worker"; +import createPoke from "./workers/poke/queue"; import createRefund from "./workers/refund/queue"; import createSubscribe from "./workers/subscribe/queue"; @@ -69,6 +70,7 @@ const persona = createPersona( parse(pipe(string("persona url"), nonEmpty("persona url")), env.PERSONA_URL), ); const credit = createCredit(redis); +const poke = createPoke(redis); const refund = createRefund(redis); const webhook = createHook(redis); const sardine = createSardine( @@ -100,12 +102,10 @@ const api = createApi({ const activityHook = createActivityHook({ alchemy, - credit, database, - executor: keeper, onesignal, + poke, redis, - segment, }); const blockHook = createBlockHook({ alchemy, blockKey: env.ALCHEMY_BLOCK_KEY, executor: keeper, onesignal, redis }); const bridgeHook = createBridgeHook({ @@ -422,6 +422,7 @@ export const close = supervise( closeMaturity, () => credit.close(), () => hookWorker.close(), + () => poke.close(), () => refund.close(), () => segment.close(), () => subscribe.close(), diff --git a/server/test/e2e.ts b/server/test/e2e.ts index 3093c6b7ca..748761fa1c 100644 --- a/server/test/e2e.ts +++ b/server/test/e2e.ts @@ -23,6 +23,7 @@ import createSardine from "../utils/sardine"; import createSegment from "../utils/segment"; import creditWorker from "../workers/credit/worker"; import hookWorker from "../workers/hook/worker"; +import pokeWorker from "../workers/poke/worker"; import refundWorker from "../workers/refund/worker"; import subscribeWorker from "../workers/subscribe/worker"; import { connect } from "../workers/worker"; @@ -45,6 +46,7 @@ describe("e2e", () => { const segment = createSegment("segment"); const workers = [ creditWorker({ bullmq, database, onesignal }), + pokeWorker({ bullmq, onesignal, poker: privateKeyToAccount(padHex("0xb0b")), segment }), refundWorker({ bullmq, database, diff --git a/server/test/hooks/activity.test.ts b/server/test/hooks/activity.test.ts index 9c4e3f9dc9..379253bdce 100644 --- a/server/test/hooks/activity.test.ts +++ b/server/test/hooks/activity.test.ts @@ -2,52 +2,32 @@ import "../mocks/alchemy"; import "../mocks/deployments"; import sendPushNotificationMock from "../mocks/onesignal"; import "../mocks/sentry"; -import "../mocks/wallet"; -import { captureException, setUser, startSpan } from "@sentry/node"; +import { captureException, setUser } from "@sentry/node"; import { testClient } from "hono/testing"; import { Redis } from "ioredis"; import { createHmac } from "node:crypto"; -import { - BaseError, - ContractFunctionRevertedError, - encodeErrorResult, - hexToBigInt, - hexToBytes, - padHex, - parseEther, - WaitForTransactionReceiptTimeoutError, - zeroHash, - type Address, - type PrivateKeyAccount, - type withRetry, -} from "viem"; +import { hexToBytes, padHex, zeroHash, type Address, type PrivateKeyAccount } from "viem"; import { generatePrivateKey, privateKeyToAccount } from "viem/accounts"; import { afterEach, beforeAll, beforeEach, describe, expect, inject, it, vi } from "vitest"; import deriveAddress from "@exactly/common/deriveAddress"; -import { exaAccountFactoryAbi, previewerAbi } from "@exactly/common/generated/chain"; import database, { credentials } from "../../database"; import activity from "../../hooks/activity"; import t, { f } from "../../i18n"; import createAlchemy, { activityUrl, NETWORKS, type Webhook } from "../../utils/alchemy"; import createOnesignal from "../../utils/onesignal"; -import publicClient from "../../utils/publicClient"; import redis from "../../utils/redis"; -import createSegment from "../../utils/segment"; -import wallet from "../../utils/wallet"; -import anvilClient from "../anvilClient"; -const executor = privateKeyToAccount(padHex("0x69")); -const waitForReceipt = publicClient.waitForTransactionReceipt; +import type createPoke from "../../workers/poke/queue"; + const alchemy = createAlchemy("webhooks"); -const credit = { - close: vi.fn<() => Promise>().mockResolvedValue(), - enqueue: vi.fn<(account: Address) => Promise>().mockResolvedValue(), -}; const onesignal = createOnesignal("onesignal"); -const segment = createSegment("segment"); +const poke = { + close: vi.fn["close"]>().mockResolvedValue(), + enqueue: vi.fn["enqueue"]>().mockResolvedValue(), +}; const activityHook = createHook(); const client = testClient(activityHook.app); @@ -68,20 +48,14 @@ const appClient = { beforeAll(() => activityHook.ready); -describe("address activity", { timeout: 66_666 }, () => { - let keeper: ReturnType; +describe("address activity", () => { let owner: PrivateKeyAccount; let account: Address; beforeEach(async () => { - credit.enqueue.mockReset().mockResolvedValue(); - keeper = wallet(executor); - vi.mocked(wallet).mockReset().mockReturnValue(keeper); + poke.enqueue.mockReset().mockResolvedValue(); owner = privateKeyToAccount(generatePrivateKey()); account = deriveAddress(inject("ExaAccountFactory"), { x: padHex(owner.address), y: zeroHash }); - vi.spyOn(publicClient, "waitForTransactionReceipt").mockImplementation((parameters) => - waitForReceipt({ ...parameters, checkReplacement: false, pollingInterval: 10 }), - ); await database.insert(credentials).values([ { @@ -112,6 +86,7 @@ describe("address activity", { timeout: 66_666 }, () => { expect(response.status).toBe(500); expect(errorConsole).toHaveBeenCalledWith(expect.objectContaining({ message: "unsupported activity network" })); + expect(poke.enqueue).not.toHaveBeenCalled(); }); it("ignores transfers to unknown accounts", async () => { @@ -126,435 +101,159 @@ describe("address activity", { timeout: 66_666 }, () => { }); expect(response.status).toBe(200); + expect(poke.enqueue).not.toHaveBeenCalled(); expect(sendPushNotification).not.toHaveBeenCalled(); }); - it("captures no balance once after retries", async () => { - vi.spyOn(keeper, "exaSend").mockImplementation((spanOptions) => - Promise.resolve( - spanOptions.op === "exa.poke" ? null : ({ status: "success" } as Awaited>), - ), - ); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, - }); - - await waitForActivity(); - - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(1); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "error")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("fails with unexpected error", async () => { - const chain = NETWORKS.get("ANVIL"); - if (!chain) throw new Error("missing anvil"); - const current = wallet(executor, chain); - const getCode = vi.fn().mockRejectedValueOnce(new Error("Unexpected")); - const createWallet = vi.mocked(wallet); - createWallet.mockClear(); - createWallet.mockReturnValueOnce({ ...current, getCode }); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); + it("omits the formatted amount when value is 0", async () => { + const sendPushNotification = sendPushNotificationMock; + const chain = NETWORKS.get("ETH_MAINNET"); + if (!chain) throw new Error("missing mainnet"); + mockLifiTokens({ 1: [{ address: inject("WETH") }] }); const response = await appClient.index.$post({ ...activityPayload, json: { ...activityPayload.json, + webhookId: "ETH_MAINNET", event: { ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], + network: "ETH_MAINNET", + activity: [ + { + ...activityPayload.json.event.activity[1], + toAddress: account, + value: 0, + rawContract: { address: inject("WETH") as Address, rawValue: "0x1" }, + }, + ], }, }, }); - await waitForActivity(); - - expect(getCode).toHaveBeenCalledOnce(); - expect(captureException).toHaveBeenCalledWith(new Error("Unexpected"), expect.objectContaining({ level: "error" })); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("fails with transaction timeout", async () => { - const exaSend = keeper.exaSend; - const poke = vi - .fn() - .mockRejectedValue(new WaitForTransactionReceiptTimeoutError({ hash: zeroHash })); - vi.spyOn(keeper, "exaSend").mockImplementation((spanOptions, call, options) => - spanOptions.op === "exa.poke" ? poke(spanOptions, call, options) : exaSend(spanOptions, call, options), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, + expect(poke.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + assets: [inject("WETH")], + chainId: chain.id, + factory: inject("ExaAccountFactory"), + origin: "activity", + publicKey: owner.address.toLowerCase(), + source: null, }); - - await waitForActivity(); - - expect(poke).toHaveBeenCalledTimes(6); - expect(captureException).toHaveBeenCalledWith( - expect.objectContaining({ name: "WaitForTransactionReceiptTimeoutError" }), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "unknown"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("fingerprints poke revert by error name", async () => { - const revertAbi = [{ type: "error", name: "Unauthorized", inputs: [] }] as const; - failPoke( - keeper, - new BaseError("test", { - cause: new ContractFunctionRevertedError({ - abi: revertAbi, - data: encodeErrorResult({ abi: revertAbi, errorName: "Unauthorized" }), - functionName: "poke", - }), - }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, + await vi.waitUntil(() => sendPushNotification.mock.calls.length > 0, 5000); + expect(sendPushNotification).toHaveBeenCalledWith({ + userId: account, + headings: t("Funds received"), + contents: t("{{amount}} received", { amount: "WETH" }), }); - - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "Unauthorized"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); expect(response.status).toBe(200); }); - it("fingerprints poke revert by reason", async () => { - failPoke( - keeper, - new BaseError("test", { - cause: new ContractFunctionRevertedError({ abi: [], functionName: "poke", message: "custom reason" }), - }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - + it("queues eth when raw value is missing", async () => { const response = await appClient.index.$post({ ...activityPayload, json: { ...activityPayload.json, event: { ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], + activity: [{ ...activityPayload.json.event.activity[0], toAddress: account, rawContract: {} }], }, }, }); - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "custom reason"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("fingerprints poke revert as unknown", async () => { - failPoke( - keeper, - new BaseError("test", { cause: new ContractFunctionRevertedError({ abi: [], functionName: "poke" }) }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, + expect(poke.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + assets: ["0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE"], + chainId: 31_337, + factory: inject("ExaAccountFactory"), + origin: "activity", + publicKey: owner.address.toLowerCase(), + source: null, }); - - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "unknown"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); expect(response.status).toBe(200); }); - it("fingerprints poke revert by signature", async () => { - failPoke( - keeper, - new BaseError("test", { - cause: new ContractFunctionRevertedError({ - abi: [], - data: "0xdeadbeef", - functionName: "poke", - }), - }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - + it("queues eth when raw value is empty", async () => { const response = await appClient.index.$post({ ...activityPayload, json: { ...activityPayload.json, event: { ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], + activity: [ + { ...activityPayload.json.event.activity[0], toAddress: account, rawContract: { rawValue: "0x" } }, + ], }, }, }); - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "0xdeadbeef"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("fingerprints shouldRetry by error name", async () => { - const revertAbi = [{ type: "error", name: "Unauthorized", inputs: [] }] as const; - failPoke( - keeper, - new BaseError("test", { - cause: new ContractFunctionRevertedError({ - abi: revertAbi, - data: encodeErrorResult({ abi: revertAbi, errorName: "Unauthorized" }), - functionName: "pokeETH", - }), - }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, + expect(poke.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + assets: ["0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE"], + chainId: 31_337, + factory: inject("ExaAccountFactory"), + origin: "activity", + publicKey: owner.address.toLowerCase(), + source: null, }); - - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "Unauthorized"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); expect(response.status).toBe(200); }); - it("fingerprints shouldRetry by reason", async () => { - failPoke( - keeper, - new BaseError("test", { - cause: new ContractFunctionRevertedError({ abi: [], functionName: "pokeETH", message: "custom reason" }), - }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - + it("queues eth when value is missing", async () => { + const { value: _, ...transfer } = activityPayload.json.event.activity[0]; const response = await appClient.index.$post({ ...activityPayload, json: { ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, + event: { ...activityPayload.json.event, activity: [{ ...transfer, toAddress: account }] }, }, }); - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "custom reason"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("fingerprints shouldRetry by signature", async () => { - failPoke( - keeper, - new BaseError("test", { - cause: new ContractFunctionRevertedError({ abi: [], data: "0xdeadbeef", functionName: "pokeETH" }), - }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, + expect(poke.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + assets: ["0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE"], + chainId: 31_337, + factory: inject("ExaAccountFactory"), + origin: "activity", + publicKey: owner.address.toLowerCase(), + source: null, }); - - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "0xdeadbeef"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); expect(response.status).toBe(200); }); - it("fingerprints shouldRetry as unknown revert", async () => { - failPoke( - keeper, - new BaseError("test", { cause: new ContractFunctionRevertedError({ abi: [], functionName: "pokeETH" }) }), - ); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - + it("queues tokens when value is missing", async () => { + const { value: _, ...transfer } = activityPayload.json.event.activity[1]; const response = await appClient.index.$post({ ...activityPayload, json: { ...activityPayload.json, event: { ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], + activity: [ + { + ...transfer, + toAddress: account, + rawContract: { ...transfer.rawContract, address: inject("WETH") }, + }, + ], }, }, }); - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.any(BaseError), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "unknown"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("fingerprints shouldRetry as unknown", async () => { - failPoke(keeper, new Error("unexpected")); - - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, + expect(poke.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + assets: [inject("WETH")], + chainId: 31_337, + factory: inject("ExaAccountFactory"), + origin: "activity", + publicKey: owner.address.toLowerCase(), + source: null, }); - - await waitForWETHMarket(account, deposit); - - expect(captureException).toHaveBeenCalledWith( - expect.objectContaining({ message: "unexpected" }), - expect.objectContaining({ level: "error", fingerprint: ["{{ default }}", "unknown"] }), - ); - expect( - vi.mocked(captureException).mock.calls.filter(([error, hint]) => isNoBalance(error, hint, "warning")), - ).toHaveLength(0); - expect(setUser).toHaveBeenCalledWith({ id: account }); expect(response.status).toBe(200); }); it("ignores zero raw values when value is missing", async () => { - const sendPushNotification = sendPushNotificationMock; const { value: _, ...transfer } = activityPayload.json.event.activity[1]; - const response = await appClient.index.$post({ ...activityPayload, json: { @@ -566,140 +265,18 @@ describe("address activity", { timeout: 66_666 }, () => { }, }); - expect(response.status).toBe(200); - expect(sendPushNotification).not.toHaveBeenCalled(); - }); - - it("pokes eth and queues credit", async () => { - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], - }, - }, - }); - const market = await waitForWETHMarket(account, deposit); - await vi.waitUntil(() => credit.enqueue.mock.calls.some(([queued]) => queued === account)); - - expect(market.floatingDepositAssets).toBe(deposit); - expect(market.isCollateral).toBe(true); - expect(credit.enqueue.mock.calls.filter(([queued]) => queued === account)).toStrictEqual([[account]]); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("pokes eth with value when rawValue is missing", async () => { - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[0], toAddress: account, rawContract: {} }], - }, - }, - }); - const market = await waitForWETHMarket(account, deposit); - - expect(market.floatingDepositAssets).toBe(deposit); - expect(market.isCollateral).toBe(true); - expect(response.status).toBe(200); - }); - - it("pokes eth with value when rawValue is 0x", async () => { - const exaSend = vi.spyOn(keeper, "exaSend"); - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [ - { ...activityPayload.json.event.activity[0], toAddress: account, rawContract: { rawValue: "0x" } }, - ], - }, - }, - }); - const market = await waitForWETHMarket(account, deposit); - - expect( - exaSend.mock.calls.some( - ([spanOptions, request]) => - spanOptions.op === "exa.poke" && - request.address === account && - "functionName" in request && - request.functionName === "pokeETH", - ), - ).toBe(true); - expect(market.floatingDepositAssets).toBe(deposit); - expect(market.isCollateral).toBe(true); - expect(response.status).toBe(200); - }); - - it("pokes eth without value", async () => { - const exaSend = vi.spyOn(keeper, "exaSend"); - const deposit = parseEther("5"); - await anvilClient.setBalance({ address: account, value: deposit }); - - const eth = activityPayload.json.event.activity[0]; - const transfer = { - fromAddress: eth.fromAddress, - toAddress: account, - hash: eth.hash, - asset: eth.asset, - category: eth.category, - rawContract: eth.rawContract, - }; - expect("value" in transfer).toBe(false); - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [transfer], - }, - }, - }); - const market = await waitForWETHMarket(account, deposit); - - expect( - exaSend.mock.calls.some( - ([spanOptions, request]) => - spanOptions.op === "exa.poke" && - request.address === account && - "functionName" in request && - request.functionName === "pokeETH", - ), - ).toBe(true); - expect(market.floatingDepositAssets).toBe(deposit); - expect(market.isCollateral).toBe(true); + expect(poke.enqueue).not.toHaveBeenCalled(); expect(response.status).toBe(200); }); - it("pokes weth and eth", async () => { - const eth = parseEther("5"); - await anvilClient.setBalance({ address: account, value: eth }); - - const weth = parseEther("2"); - await anvilClient.writeContract({ - account: null, - address: inject("WETH"), - abi: mockERC20Abi, - functionName: "mint", - args: [account, weth], + it("queues one job per account with unique assets", async () => { + const secondOwner = privateKeyToAccount(generatePrivateKey()); + const secondAccount = deriveAddress(inject("ExaAccountFactory"), { x: padHex(secondOwner.address), y: zeroHash }); + await database.insert(credentials).values({ + id: secondAccount, + publicKey: new Uint8Array(hexToBytes(secondOwner.address)), + account: secondAccount, + factory: inject("ExaAccountFactory"), }); const response = await appClient.index.$post({ @@ -715,273 +292,63 @@ describe("address activity", { timeout: 66_666 }, () => { toAddress: account, rawContract: { ...activityPayload.json.event.activity[1].rawContract, address: inject("WETH") }, }, - ], - }, - }, - }); - const market = await waitForWETHMarket(account, eth + weth); - - expect(market.floatingDepositAssets).toBe(eth + weth); - expect(market.isCollateral).toBe(true); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("pokes token without value", async () => { - const exaSend = vi.spyOn(keeper, "exaSend"); - const weth = parseEther("2"); - await anvilClient.writeContract({ - account: null, - address: inject("WETH"), - abi: mockERC20Abi, - functionName: "mint", - args: [account, weth], - }); - - const token = activityPayload.json.event.activity[1]; - const transfer = { - fromAddress: token.fromAddress, - toAddress: account, - hash: token.hash, - asset: token.asset, - category: token.category, - rawContract: { ...token.rawContract, address: inject("WETH") }, - }; - expect("value" in transfer).toBe(false); - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [transfer], - }, - }, - }); - const market = await waitForWETHMarket(account, weth); - - expect( - exaSend.mock.calls.some( - ([spanOptions, request]) => - spanOptions.op === "exa.poke" && - request.address === account && - "functionName" in request && - request.functionName === "poke", - ), - ).toBe(true); - expect(market.floatingDepositAssets).toBe(weth); - expect(market.isCollateral).toBe(true); - expect(response.status).toBe(200); - }); - - it("ignores token without value and zero rawValue", async () => { - const exaSend = vi.spyOn(keeper, "exaSend"); - const sendPushNotification = sendPushNotificationMock; - - const token = activityPayload.json.event.activity[1]; - const transfer = { - fromAddress: token.fromAddress, - toAddress: account, - hash: token.hash, - asset: token.asset, - category: token.category, - rawContract: { address: inject("WETH"), rawValue: "0x0" as const }, - }; - expect("value" in transfer).toBe(false); - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [transfer], - }, - }, - }); - await vi.waitUntil(() => exaSend.mock.calls.length > 0, 333).catch(() => undefined); - - expect( - exaSend.mock.calls.some( - ([spanOptions, request]) => - spanOptions.op === "exa.poke" && - request.address === account && - "functionName" in request && - request.functionName === "poke", - ), - ).toBe(false); - expect(sendPushNotification).not.toHaveBeenCalled(); - expect(response.status).toBe(200); - }); - - it("pokes multiple accounts", async () => { - const deposit = parseEther("5"); - const owners = [ - owner, - privateKeyToAccount(generatePrivateKey()), - privateKeyToAccount(generatePrivateKey()), - ] as const; - const accounts = owners.map(({ address }) => - deriveAddress(inject("ExaAccountFactory"), { x: padHex(address), y: zeroHash }), - ); - await Promise.all([ - ...owners.slice(1).map(({ address }, index) => { - const credential = accounts[index + 1]; - if (!credential) throw new Error("missing account"); - return database.insert(credentials).values({ - id: credential, - publicKey: new Uint8Array(hexToBytes(address)), - account: credential, - factory: inject("ExaAccountFactory"), - }); - }), - ...accounts.map((address) => anvilClient.setBalance({ address, value: deposit })), - keeper.exaSend( - { name: "create account", op: "exa.account" }, - { - address: inject("ExaAccountFactory"), - abi: exaAccountFactoryAbi, - functionName: "createAccount", - args: [0n, [{ x: hexToBigInt(owners[0].address), y: 0n }]], - }, - ), - ]); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: accounts.map((toAddress) => ({ ...activityPayload.json.event.activity[0], toAddress })), - }, - }, - }); - await Promise.all(accounts.map((address) => waitForWETHMarket(address, deposit))); - - expect(setUser).not.toHaveBeenCalled(); - expect(response.status).toBe(200); - }); - - it("deploy account for non market asset", async () => { - const [response] = await Promise.all([ - appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - event: { - ...activityPayload.json.event, - activity: [{ ...activityPayload.json.event.activity[2], toAddress: account }], - }, - }, - }), - vi.waitUntil(async () => !!(await publicClient.getCode({ address: account })), 26_666), - ]); - - const deployed = !!(await publicClient.getCode({ address: account })); - - expect(deployed).toBe(true); - expect(setUser).toHaveBeenCalledWith({ id: account }); - expect(response.status).toBe(200); - }); - - it("deploys on the event network without claiming yield", async () => { - const sendPushNotification = sendPushNotificationMock; - const chain = NETWORKS.get("ETH_MAINNET"); - if (!chain) throw new Error("missing mainnet"); - const eventWallet = wallet(executor, chain); - const getCode = vi.fn().mockResolvedValue(undefined); // eslint-disable-line unicorn/no-useless-undefined -- absent code - const eventExaSend = vi.fn().mockResolvedValue(null); - const createWallet = vi.mocked(wallet); - createWallet.mockClear(); - createWallet.mockReturnValueOnce({ ...eventWallet, getCode, exaSend: eventExaSend }); - const keeperSend = vi.spyOn(keeper, "exaSend"); - mockLifiTokens({ 1: [{ address: inject("WETH") }] }); - - const response = await appClient.index.$post({ - ...activityPayload, - json: { - ...activityPayload.json, - webhookId: "ETH_MAINNET", - event: { - ...activityPayload.json.event, - network: "ETH_MAINNET", - activity: [ { ...activityPayload.json.event.activity[1], toAddress: account, - rawContract: { address: inject("WETH") as Address, rawValue: "0x1" }, + rawContract: { ...activityPayload.json.event.activity[1].rawContract, address: inject("WETH") }, }, + { ...activityPayload.json.event.activity[0], toAddress: secondAccount }, ], }, }, }); - await vi.waitUntil(() => eventExaSend.mock.calls.length > 0); - - expect(getCode).toHaveBeenCalledWith({ address: account }); - expect(createWallet).toHaveBeenCalledWith(expect.anything(), chain); - expect(eventExaSend).toHaveBeenCalledWith( - expect.objectContaining({ attributes: { account }, name: "create account", op: "exa.account" }), - expect.objectContaining({ - abi: exaAccountFactoryAbi, - address: inject("ExaAccountFactory") as Address, - functionName: "createAccount", - }), - { fees: "auto" }, - ); - expect(keeperSend.mock.calls.some(([options]) => options.attributes?.account === account)).toBe(false); - await vi.waitUntil(() => sendPushNotification.mock.calls.length > 0, 5000); - expect(sendPushNotification).toHaveBeenCalledWith({ - userId: account, - headings: t("Funds received"), - contents: t("{{amount}} received", { amount: { en: "99.973 WETH", es: "99,973 WETH", pt: "99,973 WETH" } }), + expect(poke.enqueue).toHaveBeenCalledTimes(2); + expect(poke.enqueue).toHaveBeenNthCalledWith(1, { + account, + assets: ["0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", inject("WETH")], + chainId: 31_337, + factory: inject("ExaAccountFactory"), + origin: "activity", + publicKey: owner.address.toLowerCase(), + source: null, + }); + expect(poke.enqueue).toHaveBeenNthCalledWith(2, { + account: secondAccount, + assets: ["0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE"], + chainId: 31_337, + factory: inject("ExaAccountFactory"), + origin: "activity", + publicKey: secondOwner.address.toLowerCase(), + source: null, }); + expect(setUser).not.toHaveBeenCalled(); expect(response.status).toBe(200); }); - it("omits the formatted amount when value is 0", async () => { - const sendPushNotification = sendPushNotificationMock; - mockLifiTokens({ 1: [{ address: inject("WETH") }] }); + it("fails the webhook when poke cannot be queued", async () => { + const error = new Error("redis unavailable"); + const errorConsole = vi.spyOn(console, "error").mockImplementation(() => undefined); + poke.enqueue.mockRejectedValueOnce(error); const response = await appClient.index.$post({ ...activityPayload, json: { ...activityPayload.json, - webhookId: "ETH_MAINNET", event: { ...activityPayload.json.event, - network: "ETH_MAINNET", - activity: [ - { - ...activityPayload.json.event.activity[1], - toAddress: account, - value: 0, - rawContract: { address: inject("WETH") as Address, rawValue: "0x1" }, - }, - ], + activity: [{ ...activityPayload.json.event.activity[0], toAddress: account }], }, }, }); - await vi.waitUntil(() => sendPushNotification.mock.calls.length > 0, 5000); - expect(sendPushNotification).toHaveBeenCalledWith({ - userId: account, - headings: t("Funds received"), - contents: t("{{amount}} received", { amount: "WETH" }), - }); - expect(response.status).toBe(200); + expect(response.status).toBe(500); + expect(errorConsole).toHaveBeenCalledWith(error); + expect(poke.enqueue).toHaveBeenCalledOnce(); }); it("sends translated notification without symbol when asset is missing", async () => { const sendPushNotification = sendPushNotificationMock; - const amount = parseEther(String(activityPayload.json.event.activity[1].value)); - await anvilClient.writeContract({ - account: null, - address: inject("WETH"), - abi: mockERC20Abi, - functionName: "mint", - args: [account, amount], - }); const { asset: _, ...tokenWithoutAsset } = activityPayload.json.event.activity[1]; const response = await appClient.index.$post({ @@ -1001,10 +368,7 @@ describe("address activity", { timeout: 66_666 }, () => { }, }); - await Promise.all([ - vi.waitUntil(() => sendPushNotification.mock.calls.length > 0), - waitForWETHMarket(account, amount), - ]); + await vi.waitUntil(() => sendPushNotification.mock.calls.length > 0); expect(sendPushNotification).toHaveBeenCalledWith({ userId: account, @@ -1017,14 +381,6 @@ describe("address activity", { timeout: 66_666 }, () => { it("captures funds received notification errors", async () => { const error = new Error("push failed"); sendPushNotificationMock.mockRejectedValueOnce(error); - const amount = parseEther(String(activityPayload.json.event.activity[1].value)); - await anvilClient.writeContract({ - account: null, - address: inject("WETH"), - abi: mockERC20Abi, - functionName: "mint", - args: [account, amount], - }); const response = await appClient.index.$post({ ...activityPayload, @@ -1043,10 +399,7 @@ describe("address activity", { timeout: 66_666 }, () => { }, }); - await Promise.all([ - vi.waitUntil(() => vi.mocked(captureException).mock.calls.some(([captured]) => captured === error)), - waitForWETHMarket(account, amount), - ]); + await vi.waitUntil(() => vi.mocked(captureException).mock.calls.some(([captured]) => captured === error)); expect(captureException).toHaveBeenCalledWith(error, { level: "error" }); expect(response.status).toBe(200); @@ -1389,64 +742,6 @@ describe("webhook authentication", () => { }); }); -function failPoke(keeper: ReturnType, error: Error) { - const { exaSend } = keeper; - let failed = false; - vi.spyOn(keeper, "exaSend").mockImplementation((spanOptions, call, options) => { - if (failed || spanOptions.op !== "exa.poke") return exaSend(spanOptions, call, options); - failed = true; - return Promise.reject(error); - }); -} - -async function getWETHMarket(account: Address) { - const exactly = await publicClient.readContract({ - address: inject("Previewer"), - functionName: "exactly", - abi: previewerAbi, - args: [account], - }); - return exactly.find((market) => market.asset === inject("WETH")); -} - -async function waitForWETHMarket(account: Address, floatingDepositAssets: bigint) { - await waitForActivity(); - return vi.waitUntil(async () => { - try { - const market = await getWETHMarket(account); - if (!market) return false; - return market.floatingDepositAssets === floatingDepositAssets && market.isCollateral ? market : false; - } catch (error) { - if ( - error instanceof BaseError && - error.shortMessage.includes("Arithmetic operation resulted in underflow or overflow.") - ) - return false; - throw error; - } - }, 26_666); -} - -async function waitForActivity() { - const spans = vi.mocked(startSpan); - const pending = spans.mock.calls.flatMap(([options], index) => - options.op === "exa.activity" ? [spans.mock.results[index]?.value as unknown] : [], - ); - expect(pending).not.toHaveLength(0); - await Promise.allSettled(pending); -} - -function isNoBalance(error: unknown, hint: unknown, level: "error" | "warning") { - const data = hint as Record | undefined; - return ( - error instanceof Error && - error.message === "NoBalance()" && - data?.level === level && - Array.isArray(data.fingerprint) && - data.fingerprint.join(":") === "{{ default }}:NoBalance" - ); -} - function mockLifiTokens(response: Error | Record | Response) { const originalFetch = globalThis.fetch; vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { @@ -1461,19 +756,10 @@ function mockLifiTokens(response: Error | Record }); } -const mockERC20Abi = [ - { - type: "function", - name: "mint", - inputs: [{ type: "address" }, { type: "uint256" }], - outputs: [], - stateMutability: "nonpayable", - }, -] as const; - const activityPayload = { header: {}, json: { + id: "event", type: "ADDRESS_ACTIVITY", webhookId: "activity", event: { @@ -1522,33 +808,19 @@ const activityPayload = { vi.mock("@account-kit/infra", { spy: true }); vi.mock("@sentry/node", { spy: true }); -vi.mock("viem", async (importOriginal) => { - const original = await importOriginal<{ withRetry: typeof withRetry }>(); - return { - ...original, - withRetry: ( - callback: Parameters[0], - options: Parameters[1], - ) => original.withRetry(callback, { ...options, delay: 1 }), - }; -}); - -afterEach(async () => { - if (vi.mocked(startSpan).mock.calls.some(([options]) => options.op === "exa.activity")) await waitForActivity(); +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); vi.restoreAllMocks(); -}, 66_666); +}); function createHook(current = alchemy) { return activity({ alchemy: current, - credit, database, - executor, onesignal, + poke, redis, - segment, }); } diff --git a/server/test/workers/bin.test.ts b/server/test/workers/bin.test.ts index b05593ca74..e67293c8f1 100644 --- a/server/test/workers/bin.test.ts +++ b/server/test/workers/bin.test.ts @@ -9,6 +9,7 @@ const refunder = privateKeyToAccount(padHex("0xfee")); const database = { $client: { end: vi.fn<() => Promise>() } }; const onesignal = {}; const panda = {}; +const poker = privateKeyToAccount(padHex("0xb0b")); const sardine = {}; const segment = { close: vi.fn<() => Promise>() }; const whatsapp = {}; @@ -21,6 +22,9 @@ const mocks = { hook: vi.fn<(config: { bullmq: object; database: typeof database; panda: object }) => Handle>(), onesignal: vi.fn<(key: string) => object>(), panda: vi.fn<(config: { key: string; url: string }) => object>(), + poke: vi.fn< + (config: { bullmq: object; onesignal: object; poker: typeof poker; segment: typeof segment }) => Handle + >(), refund: vi.fn< (config: { @@ -55,12 +59,13 @@ beforeEach(() => { mocks.hook.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.onesignal.mockReset().mockReturnValue(onesignal); mocks.panda.mockReset().mockReturnValue(panda); + mocks.poke.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.chat.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.refund.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.sardine.mockReset().mockReturnValue(sardine); mocks.secret.mockReset().mockImplementation((name) => Promise.resolve(name)); mocks.segment.mockReset().mockReturnValue(segment); - mocks.signer.mockReset().mockResolvedValue(refunder); + mocks.signer.mockReset().mockImplementation((name) => Promise.resolve(name === "poker" ? poker : refunder)); mocks.subscribe.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.supervise.mockReset(); mocks.whatsapp.mockReset().mockReturnValue(whatsapp); @@ -88,6 +93,7 @@ beforeEach(() => { vi.doMock("../../workers/chat/worker", () => ({ default: mocks.chat })); vi.doMock("../../workers/credit/worker", () => ({ default: mocks.credit })); vi.doMock("../../workers/hook/worker", () => ({ default: mocks.hook })); + vi.doMock("../../workers/poke/worker", () => ({ default: mocks.poke })); vi.doMock("../../workers/refund/worker", () => ({ default: mocks.refund })); vi.doMock("../../workers/subscribe/worker", () => ({ default: mocks.subscribe })); }); @@ -138,6 +144,46 @@ describe("bin", () => { }); }); + it("resolves poke private config before constructing and supervising its worker", async () => { + await import("../../workers/poke/bin"); + + const created = mocks.supervise.mock.calls[0]?.[1]; + if (!created) throw new Error("missing worker"); + expect(mocks.supervise).toHaveBeenCalledExactlyOnceWith("poke", created); + await created; + expect(mocks.secret.mock.calls.map(([secret]) => secret)).toStrictEqual([ + "redis-url", + "poke-onesignal-api-key", + "poke-segment-write-key", + ]); + expect(new Set(mocks.secret.mock.calls.map(([, secrets]) => secrets)).size).toBe(1); + expect(mocks.onesignal).toHaveBeenCalledExactlyOnceWith("poke-onesignal-api-key"); + expect(mocks.segment).toHaveBeenCalledExactlyOnceWith("poke-segment-write-key"); + expect(mocks.signer.mock.calls.map(([signer]) => signer)).toStrictEqual(["poker"]); + expect(mocks.poke).toHaveBeenCalledExactlyOnceWith({ + bullmq: expect.objectContaining({ redisUrl: "redis-url", options: { maxRetriesPerRequest: null } }) as object, + onesignal, + poker, + segment, + }); + }); + + it("fails before constructing the poke worker without its poker account", async () => { + const error = new Error("missing poker"); + mocks.signer.mockRejectedValueOnce(error); + mocks.supervise.mockImplementation((_, created) => { + created.catch(() => undefined); + }); + + await import("../../workers/poke/bin"); + const created = mocks.supervise.mock.calls[0]?.[1]; + if (!created) throw new Error("missing worker"); + + await expect(created).rejects.toBe(error); + expect(mocks.signer.mock.calls.map(([signer]) => signer)).toStrictEqual(["poker"]); + expect(mocks.poke).not.toHaveBeenCalled(); + }); + it("resolves refund private config before constructing and supervising its worker", async () => { await import("../../workers/refund/bin"); diff --git a/server/test/workers/poke.test.ts b/server/test/workers/poke.test.ts new file mode 100644 index 0000000000..764cbcfcd3 --- /dev/null +++ b/server/test/workers/poke.test.ts @@ -0,0 +1,706 @@ +import "../mocks/deployments"; +import sendPushNotificationMock from "../mocks/onesignal"; +import "../mocks/sentry"; +import "../mocks/wallet"; + +import { captureException, continueTrace, startSpan, withScope } from "@sentry/node"; +import { Queue, QueueEvents, Job as QueueJob } from "bullmq"; +import { parse } from "valibot"; +import { + BaseError, + ContractFunctionRevertedError, + encodeErrorResult, + erc20Abi, + padHex, + parseEther, + zeroHash, +} from "viem"; +import { generatePrivateKey, privateKeyToAccount } from "viem/accounts"; +import { afterAll, beforeAll, beforeEach, describe, expect, inject, it, vi } from "vitest"; + +import deriveAddress from "@exactly/common/deriveAddress"; +import chain, { previewerAbi, wethAddress } from "@exactly/common/generated/chain"; +import { Address } from "@exactly/common/validation"; + +import t from "../../i18n"; +import { NETWORKS } from "../../utils/alchemy"; +import createOnesignal from "../../utils/onesignal"; +import publicClient from "../../utils/publicClient"; +import { bullmq } from "../../utils/redis"; +import createSegment from "../../utils/segment"; +import wallet from "../../utils/wallet"; +import createPoke from "../../workers/poke/queue"; +import pokeWorker from "../../workers/poke/worker"; +import anvilClient from "../anvilClient"; + +import type { Job as Credit } from "../../workers/credit/job"; +import type { Job as Poke } from "../../workers/poke/job"; +import type * as sentry from "@sentry/node"; +import type { JobsOptions } from "bullmq"; + +const eth = parse(Address, "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); +const token = parse(Address, wethAddress); +const token2 = parse(Address, inject("USDC")); +const unknownAsset = parse(Address, "0x3333333333333333333333333333333333333333"); +const weth = parse(Address, wethAddress); +const poker = privateKeyToAccount(padHex("0xb0b")); +const poke = createPoke(bullmq); +const mocks = vi.hoisted(() => ({ + closeSegment: vi.fn(), + segmentOn: vi.fn<(event: string, listener: (error: Error) => void) => void>(), + track: vi.fn(), +})); + +vi.mock("@segment/analytics-node", () => ({ + Analytics: class { + closeAndFlush = mocks.closeSegment; + on = mocks.segmentOn; + track = mocks.track; + }, +})); + +const onesignal = createOnesignal("onesignal"); +let segment: ReturnType; +let account: Address; +let request: Parameters["enqueue"]>[0]; + +const credits = new Queue("credit", { connection: bullmq }); +const queue = new Queue("poke", { connection: bullmq }); +const events = new QueueEvents("poke", { connection: bullmq }); +let worker: Awaited>; +let segmentError: ((error: Error) => void) | undefined; +let segmentEvent: string | undefined; + +async function jobFinished( + current: Parameters["enqueue"]>[0], + options?: JobsOptions, + trace?: Pick, +) { + const id = [current.chainId, current.account, ...(current.assets ?? [])].join("-"); + const job = await queue.add( + "poke", + { ...current, ...trace }, + { attempts: 1, jobId: id, removeOnComplete: true, removeOnFail: true, ...options }, + ); + await job.waitUntilFinished(events).catch(async (error: unknown) => { + await vi.waitUntil(() => vi.mocked(captureException).mock.calls.length > 0); + throw error; + }); + return job; +} + +async function spyScopeSetUser() { + const { withScope: realWithScope } = await vi.importActual("@sentry/node"); + const setUser = vi.fn(); + vi.mocked(withScope).mockImplementation((_scopeOrCallback, _callback?) => + realWithScope((scope) => { + const originalSetUser = scope.setUser.bind(scope); + scope.setUser = (...args: Parameters) => { + setUser(...args); + return originalSetUser(...args); + }; + return ((_callback ?? _scopeOrCallback) as NonNullable)(scope); + }), + ); + return setUser; +} + +async function spySpanSetAttribute() { + const { startSpan: realStartSpan } = await vi.importActual("@sentry/node"); + const setAttribute = vi.fn(); + vi.mocked(startSpan).mockImplementation(((options, callback) => + realStartSpan(options, (span) => { + const originalSetAttribute = span.setAttribute.bind(span); + span.setAttribute = (...args: Parameters) => { + setAttribute(...args); + return originalSetAttribute(...args); + }; + return callback(span); + })) as typeof startSpan); + return setAttribute; +} + +beforeEach(async () => { + vi.restoreAllMocks(); + mocks.closeSegment.mockReset().mockImplementation(() => Promise.resolve()); + mocks.segmentOn.mockReset(); + mocks.track.mockReset(); + sendPushNotificationMock.mockResolvedValue({}); + ({ account, request } = createRequest()); + const waitForReceipt = publicClient.waitForTransactionReceipt; + vi.spyOn(publicClient, "waitForTransactionReceipt").mockImplementation((parameters) => + waitForReceipt({ ...parameters, checkReplacement: false, pollingInterval: 10 }), + ); + vi.clearAllMocks(); + await queue.drain(true); + await queue.clean(0, 1000, "completed"); + await queue.clean(0, 1000, "failed"); +}); +afterAll(async () => { + await Promise.all([credits.close(), events.close(), queue.close(), poke.close()]); +}); + +describe("poke queue", () => { + it("publishes account poke jobs", async () => { + await expect(poke.enqueue(request)).resolves.toBeUndefined(); + + const id = `${request.chainId}-${account}`; + const job = await queue.getJob(id); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(id); + expect(job.name).toBe("poke"); + expect(job.data).toStrictEqual({ + ...request, + sentryBaggage: expect.any(String) as string, + sentryTrace: expect.any(String) as string, + }); + expect(job.opts).toStrictEqual({ + attempts: 10, + backoff: { type: "exponential", delay: 1000 }, + jobId: id, + removeOnComplete: true, + removeOnFail: { count: 1000, age: 7 * 24 * 3600 }, + }); + await expect(job.getState()).resolves.toBe("waiting"); + expect(startSpan).toHaveBeenCalledWith( + expect.objectContaining({ + name: "poke", + op: "queue.publish", + attributes: { "messaging.destination.name": "poke" }, + }), + expect.any(Function), + ); + expect(captureException).not.toHaveBeenCalled(); + await job.remove(); + }); + + it("includes assets in job ids", async () => { + await poke.enqueue({ ...request, assets: [token] }); + + const id = `${request.chainId}-${account}-${token}`; + const job = await queue.getJob(id); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(id); + expect(job.data).toStrictEqual({ + ...request, + assets: [token], + sentryBaggage: expect.any(String) as string, + sentryTrace: expect.any(String) as string, + }); + await job.remove(); + }); + + it("keeps chains separate in job ids", async () => { + await Promise.all([poke.enqueue(request), poke.enqueue({ ...request, chainId: 1 })]); + + const [current, mainnet] = await Promise.all([ + queue.getJob(`${request.chainId}-${account}`), + queue.getJob(`1-${account}`), + ]); + if (!current || !mainnet) throw new Error("job not found"); + expect([current.id, mainnet.id]).toStrictEqual([`${request.chainId}-${account}`, `1-${account}`]); + await Promise.all([current.remove(), mainnet.remove()]); + }); + + it("propagates queue failures", async () => { + const error = new Error("queue error"); + vi.spyOn(Queue.prototype, "add").mockRejectedValueOnce(error); + + await expect(poke.enqueue(request)).rejects.toThrow(error); + + expect(captureException).not.toHaveBeenCalled(); + }); +}); + +describe("poke worker", () => { + beforeAll(async () => { + mocks.segmentOn.mockImplementationOnce((event, listener) => { + segmentError = listener; + segmentEvent = event; + }); + segment = createSegment("segment"); + worker = pokeWorker({ bullmq, onesignal, poker, segment }); + await worker.ready; + }); + + afterAll(() => worker.close()); + + it("captures segment errors", () => { + const error = new Error("segment error"); + if (!segmentError) throw new Error("missing segment error handler"); + + segmentError(error); + + expect(segmentEvent).toBe("error"); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { level: "error" }); + }); + + it("rejects unsupported chains before creating a wallet", async () => { + const readContract = vi.spyOn(publicClient, "readContract"); + const setUser = await spyScopeSetUser(); + + await expect(jobFinished({ ...request, chainId: 0 })).rejects.toThrow("unsupported chain 0"); + + expect(wallet).not.toHaveBeenCalled(); + expect(readContract).not.toHaveBeenCalled(); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + expect(setUser).toHaveBeenCalledExactlyOnceWith({ id: account }); + expect(captureException).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ message: "unsupported chain 0" }), + { + extra: { account, attempts: 1, id: `0-${account}` }, + fingerprint: ["{{ default }}", "unknown"], + level: "error", + tags: { queue: "poke", job: "poke" }, + }, + ); + }); + + it("deploys and pokes funded accounts after allow", async () => { + const deposit = parseEther("5"); + const setAttribute = await spySpanSetAttribute(); + await anvilClient.setBalance({ address: account, value: deposit }); + + await jobFinished(request); + + expect(await publicClient.getCode({ address: account })).toBeDefined(); + expect(await getWETHMarket(account)).toMatchObject({ floatingDepositAssets: deposit, isCollateral: true }); + expect(wallet).toHaveBeenCalledExactlyOnceWith(poker, NETWORKS.get("ANVIL")); + expect(mocks.track).toHaveBeenCalledWith({ event: "AccountFunded", userId: account, properties: { source: null } }); + expect(sendPushNotificationMock).toHaveBeenCalledExactlyOnceWith({ + userId: account, + headings: t("Account assets updated"), + contents: t("Your funds are ready to use"), + }); + expect(setAttribute.mock.calls.filter(([attribute]) => attribute === "exa.new")).toStrictEqual([["exa.new", true]]); + expect(startSpan).toHaveBeenCalledWith( + expect.objectContaining({ forceTransaction: true, name: "poke worker" }), + expect.any(Function), + ); + expect(startSpan).toHaveBeenCalledWith( + expect.objectContaining({ name: "poke", op: "queue.process" }), + expect.any(Function), + ); + expect(captureException).not.toHaveBeenCalled(); + }); + + it("pokes multiple accounts", async () => { + const deposit = parseEther("5"); + const requests = Array.from({ length: 3 }, createRequest); + await Promise.all( + requests.map(({ account: current }) => anvilClient.setBalance({ address: current, value: deposit })), + ); + + await Promise.all(requests.map(({ request: current }) => jobFinished(current))); + + for (const { account: current } of requests) { + expect(await getWETHMarket(current)).toMatchObject({ floatingDepositAssets: deposit, isCollateral: true }); + } + expect(captureException).not.toHaveBeenCalled(); + }); + + it("captures account funding tracking errors", async () => { + const error = new Error("tracking error"); + mocks.track.mockImplementationOnce(() => { + throw error; + }); + + await jobFinished({ ...request, assets: [unknownAsset] }); + + expect(await publicClient.getCode({ address: account })).toBeDefined(); + expect(mocks.track).toHaveBeenCalledExactlyOnceWith({ + event: "AccountFunded", + userId: account, + properties: { source: null }, + }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { level: "error" }); + }); + + it("pokes weth with eth", async () => { + const ethDeposit = parseEther("5"); + const wethDeposit = parseEther("2"); + await Promise.all([ + anvilClient.setBalance({ address: account, value: ethDeposit }), + mint(weth, account, wethDeposit), + ]); + + await jobFinished({ ...request, assets: [eth, weth] }); + + expect(await getWETHMarket(account)).toMatchObject({ + floatingDepositAssets: ethDeposit + wethDeposit, + isCollateral: true, + }); + await expect( + publicClient.readContract({ address: weth, functionName: "balanceOf", args: [account], abi: erc20Abi }), + ).resolves.toBe(0n); + }); + + it("removes weth from activity retries when eth is funded", async () => { + const ethDeposit = parseEther("5"); + const wethDeposit = parseEther("2"); + const updateData = vi.spyOn(QueueJob.prototype, "updateData"); + await Promise.all([ + anvilClient.setBalance({ address: account, value: ethDeposit }), + mint(weth, account, wethDeposit), + ]); + + await jobFinished({ ...request, assets: [eth, weth], origin: "activity" }); + + expect(updateData).toHaveBeenCalledExactlyOnceWith({ ...request, assets: [], origin: "activity" }); + expect(await getWETHMarket(account)).toMatchObject({ + floatingDepositAssets: ethDeposit + wethDeposit, + isCollateral: true, + }); + await expect( + publicClient.readContract({ address: weth, functionName: "balanceOf", args: [account], abi: erc20Abi }), + ).resolves.toBe(0n); + expect(captureException).not.toHaveBeenCalled(); + }); + + it("treats ignored no balance receipts as idempotent success", async () => { + const readContract = publicClient.readContract; + vi.spyOn(publicClient, "readContract").mockImplementation((parameters) => + parameters.functionName === "balanceOf" ? Promise.resolve(2n) : readContract(parameters as never), + ); + + await jobFinished({ ...request, assets: [token] }); + + expect(await getWETHMarket(account)).toMatchObject({ floatingDepositAssets: 0n, isCollateral: false }); + expect(mocks.track).toHaveBeenCalledExactlyOnceWith({ + event: "AccountFunded", + userId: account, + properties: { source: null }, + }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + expect(captureException).not.toHaveBeenCalled(); + }); + + it("captures notification errors without retrying", async () => { + const error = new Error("notification error"); + const deposit = parseEther("2"); + await mint(token, account, deposit); + sendPushNotificationMock.mockRejectedValueOnce(error); + + await jobFinished({ ...request, assets: [token] }); + + expect(await getWETHMarket(account)).toMatchObject({ floatingDepositAssets: deposit, isCollateral: true }); + expect(sendPushNotificationMock).toHaveBeenCalledOnce(); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { level: "error" }); + }); + + it("queues credit after activity pokes", async () => { + const deposit = parseEther("5"); + await anvilClient.setBalance({ address: account, value: deposit }); + const job = await jobFinished({ ...request, assets: [eth], origin: "activity" }); + + const credit = await credits.getJob(`poke-${job.id}`); + if (!credit) throw new Error("credit job not found"); + expect(credit.id).toBe(`poke-${job.id}`); + expect(credit.name).toBe("credit"); + expect(credit.data).toStrictEqual({ + account, + sentryBaggage: expect.any(String) as string, + sentryTrace: expect.any(String) as string, + }); + expect(credit.opts).toStrictEqual({ + attempts: 10, + backoff: { type: "exponential", delay: 1000 }, + jobId: `poke-${job.id}`, + removeOnComplete: { count: 100 }, + removeOnFail: { count: 1000, age: 7 * 24 * 3600 }, + }); + expect(await getWETHMarket(account)).toMatchObject({ floatingDepositAssets: deposit, isCollateral: true }); + await credit.remove(); + }); + + it("retries activity when credit cannot be queued", async () => { + const error = new Error("credit unavailable"); + const deposit = parseEther("5"); + const add = queue.add.bind(queue); + await anvilClient.setBalance({ address: account, value: deposit }); + vi.spyOn(Queue.prototype, "add").mockImplementation((jobName: string, data: unknown, options?: JobsOptions) => { + if (jobName === "credit") return Promise.reject(error); + return add(jobName as "poke", data as Poke, options); + }); + + await expect( + jobFinished( + { ...request, assets: [eth], origin: "activity" }, + { attempts: 2, backoff: { type: "fixed", delay: 1 } }, + ), + ).rejects.toThrow("credit unavailable"); + + expect(await getWETHMarket(account)).toMatchObject({ floatingDepositAssets: deposit, isCollateral: true }); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account, attempts: 2, id: `${request.chainId}-${account}-${eth}` }, + fingerprint: ["{{ default }}", "unknown"], + level: "error", + tags: { queue: "poke", job: "poke" }, + }); + }); + + it("treats empty balances as an idempotent success", async () => { + await jobFinished({ ...request, assets: [token] }); + + expect(await publicClient.getCode({ address: account })).toBeDefined(); + expect(await getWETHMarket(account)).toMatchObject({ floatingDepositAssets: 0n, isCollateral: false }); + expect(sendPushNotificationMock).not.toHaveBeenCalled(); + }); + + it("retries activity until its balance is visible", async () => { + const deposit = parseEther("5"); + const getBalance = publicClient.getBalance; + vi.spyOn(publicClient, "getBalance") + .mockImplementationOnce(async () => { + await anvilClient.setBalance({ address: account, value: deposit }); + return 0n; + }) + .mockImplementation((parameters) => getBalance(parameters)); + + await jobFinished( + { ...request, assets: [eth], origin: "activity" }, + { attempts: 2, backoff: { type: "fixed", delay: 1 } }, + ); + + expect(await getWETHMarket(account)).toMatchObject({ floatingDepositAssets: deposit, isCollateral: true }); + expect(captureException).not.toHaveBeenCalled(); + }); + + it("retries only activity assets that remain pending", async () => { + const wethDeposit = parseEther("2"); + const usdcDeposit = 2_000_000n; + const readContract = publicClient.readContract; + let hidden = true; + await mint(token, account, wethDeposit); + vi.spyOn(publicClient, "readContract").mockImplementation(async (parameters) => { + if (hidden && parameters.functionName === "balanceOf" && parameters.address === token2) { + hidden = false; + await mint(token2, account, usdcDeposit); + return 0n as never; + } + return readContract(parameters as never); + }); + + await jobFinished( + { ...request, assets: [token, token2], origin: "activity" }, + { attempts: 2, backoff: { type: "fixed", delay: 1 } }, + ); + + expect(hidden).toBe(false); + expect(await getMarket(account, token)).toMatchObject({ floatingDepositAssets: wethDeposit, isCollateral: true }); + expect(await getMarket(account, token2)).toMatchObject({ + floatingDepositAssets: usdcDeposit - 1n, + isCollateral: true, + }); + }); + + it("captures exhausted activity as a no balance warning", async () => { + const setUser = await spyScopeSetUser(); + const id = `${request.chainId}-${account}-${token}`; + + await expect( + jobFinished({ ...request, assets: [token], origin: "activity" }, { removeOnFail: false }), + ).rejects.toThrow("NoBalance()"); + + await expect(queue.getFailedCount()).resolves.toBe(1); + await expect(queue.getJobState(id)).resolves.toBe("failed"); + const [job] = await queue.getFailed(); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(id); + expect(job.failedReason).toBe("NoBalance()"); + expect(job.attemptsMade).toBe(1); + expect(job.stacktrace).toHaveLength(1); + expect(setUser.mock.calls).toStrictEqual([[{ id: account }], [{ id: account }]]); + expect(captureException).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ message: "NoBalance()" }), { + extra: { account, attempts: 1, id }, + fingerprint: ["{{ default }}", "NoBalance"], + level: "warning", + tags: { queue: "poke", job: "poke" }, + }); + await job.remove(); + }); + + it("deploys without poking on other chains", async () => { + const network = NETWORKS.get("ANVIL"); + if (!network) throw new Error("missing anvil"); + const readContract = vi.spyOn(publicClient, "readContract"); + const id = chain.id; + Object.assign(chain, { id: 1 }); + + try { + await jobFinished({ ...request, chainId: network.id }); + } finally { + Object.assign(chain, { id }); + } + + expect(await publicClient.getCode({ address: account })).toBeDefined(); + expect(wallet).toHaveBeenCalledExactlyOnceWith(poker, network); + expect(readContract).not.toHaveBeenCalled(); + }); + + it("deploys activity accounts funded with unsupported assets", async () => { + const job = await jobFinished({ ...request, assets: [unknownAsset], origin: "activity" }); + + expect(await publicClient.getCode({ address: account })).toBeDefined(); + expect(mocks.track).toHaveBeenCalledWith({ event: "AccountFunded", userId: account, properties: { source: null } }); + const credit = await credits.getJob(`poke-${job.id}`); + if (!credit) throw new Error("credit job not found"); + expect(credit.data.account).toBe(account); + await credit.remove(); + }); + + it("retries rpc failures", async () => { + const readContract = publicClient.readContract; + vi.spyOn(publicClient, "readContract") + .mockRejectedValueOnce(new Error("rpc unavailable")) + .mockImplementation((parameters) => readContract(parameters as never)); + + await jobFinished({ ...request, assets: [token] }, { attempts: 2, backoff: { type: "fixed", delay: 1 } }); + + expect(await publicClient.getCode({ address: account })).toBeDefined(); + expect(captureException).not.toHaveBeenCalled(); + }); + + it("captures terminal failures", async () => { + const error = new Error("poke failed"); + const setUser = await spyScopeSetUser(); + const id = `${request.chainId}-${account}-${token}`; + vi.spyOn(publicClient, "readContract").mockRejectedValue(error); + + await expect( + jobFinished( + { ...request, assets: [token] }, + { attempts: 2, backoff: { type: "fixed", delay: 1 }, removeOnFail: false }, + ), + ).rejects.toThrow("poke failed"); + + await expect(queue.getFailedCount()).resolves.toBe(1); + await expect(queue.getJobState(id)).resolves.toBe("failed"); + const [job] = await queue.getFailed(); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(id); + expect(job.failedReason).toBe("poke failed"); + expect(job.attemptsMade).toBe(2); + expect(job.stacktrace).toHaveLength(2); + expect(setUser.mock.calls).toStrictEqual([[{ id: account }], [{ id: account }]]); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account, attempts: 2, id }, + fingerprint: ["{{ default }}", "unknown"], + level: "error", + tags: { queue: "poke", job: "poke" }, + }); + await job.remove(); + }); + + it("fingerprints terminal reverts by error name", async () => { + const abi = [{ type: "error", name: "Unauthorized", inputs: [] }] as const; + const error = new BaseError("test", { + cause: new ContractFunctionRevertedError({ + abi, + data: encodeErrorResult({ abi, errorName: "Unauthorized" }), + functionName: "poke", + }), + }); + vi.spyOn(publicClient, "readContract").mockRejectedValueOnce(error); + + const setUser = await spyScopeSetUser(); + + await expect(jobFinished({ ...request, assets: [token] })).rejects.toThrow("test"); + + expect(setUser.mock.calls).toStrictEqual([[{ id: account }], [{ id: account }]]); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account, attempts: 1, id: `${request.chainId}-${account}-${token}` }, + fingerprint: ["{{ default }}", "Unauthorized"], + level: "error", + tags: { queue: "poke", job: "poke" }, + }); + }); + + it("continues sentry traces", async () => { + await jobFinished({ ...request, assets: [unknownAsset] }, undefined, { + sentryBaggage: "baggage", + sentryTrace: "trace", + }); + + expect(continueTrace).toHaveBeenCalledWith({ sentryTrace: "trace", baggage: "baggage" }, expect.any(Function)); + }); + + it("captures worker errors", () => { + const error = new Error("worker error"); + + worker.queue.emit("error", error); + + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { level: "error", tags: { queue: "poke" } }); + }); + + it("captures failed events without a job", async () => { + const error = new Error("failed event error"); + const setUser = await spyScopeSetUser(); + + worker.queue.emit("failed", undefined, error, "active"); + + expect(setUser).not.toHaveBeenCalled(); + expect(captureException).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account: undefined, attempts: undefined, id: undefined }, + fingerprint: ["{{ default }}", "unknown"], + level: "error", + tags: { queue: "poke", job: undefined }, + }); + }); + + it("skips intermediate failed events", () => { + const error = new Error("failed event error"); + + worker.queue.emit( + "failed", + { attemptsMade: 9, data: request, name: "poke", opts: {} } as unknown as Awaited>, + error, + "active", + ); + + expect(captureException).not.toHaveBeenCalled(); + }); +}); + +function createRequest() { + const owner = privateKeyToAccount(generatePrivateKey()); + const address = deriveAddress(inject("ExaAccountFactory"), { x: padHex(owner.address), y: zeroHash }); + return { + account: address, + request: { + account: address, + chainId: chain.id, + factory: inject("ExaAccountFactory"), + origin: "allow", + publicKey: owner.address, + source: null, + } satisfies Parameters["enqueue"]>[0], + }; +} + +async function getMarket(address: Address, asset: Address) { + return publicClient + .readContract({ address: inject("Previewer"), functionName: "exactly", abi: previewerAbi, args: [address] }) + .then((markets) => markets.find((current) => current.asset === asset)); +} + +const getWETHMarket = (address: Address) => getMarket(address, token); + +async function mint(asset: Address, address: Address, amount: bigint) { + await anvilClient.writeContract({ + account: null, + address: asset, + functionName: "mint", + args: [address, amount], + abi: [ + { + type: "function", + name: "mint", + inputs: [{ type: "address" }, { type: "uint256" }], + outputs: [], + stateMutability: "nonpayable", + }, + ], + }); +} diff --git a/server/workers/credit/queue.ts b/server/workers/credit/queue.ts index 45a03933bd..eeea1f707d 100644 --- a/server/workers/credit/queue.ts +++ b/server/workers/credit/queue.ts @@ -1,14 +1,15 @@ import { attempts, name, type Job } from "./job"; import createQueue from "../queue"; +import type { DefaultJobOptions } from "bullmq"; import type { Redis } from "ioredis"; -export default function queue(redis: Redis) { - const instance = createQueue(name, attempts, redis); +export default function queue(redis: Redis, removeOnComplete: DefaultJobOptions["removeOnComplete"] = true) { + const instance = createQueue(name, attempts, redis, { removeOnComplete }); return { close: () => instance.close(), - async enqueue(account: Job["account"]) { - await instance.enqueue({ account }, account); + async enqueue(account: Job["account"], jobId: string = account) { + await instance.enqueue({ account }, jobId); }, }; } diff --git a/server/workers/poke/bin.ts b/server/workers/poke/bin.ts new file mode 100644 index 0000000000..7d2231a534 --- /dev/null +++ b/server/workers/poke/bin.ts @@ -0,0 +1,32 @@ +import { KeyManagementServiceClient } from "@google-cloud/kms"; +import { SecretManagerServiceClient } from "@google-cloud/secret-manager"; + +import { name } from "./job"; +import worker from "./worker"; +import supervise, { own } from "../../supervise"; +import createOnesignal from "../../utils/onesignal"; +import secret from "../../utils/secret"; +import createSegment from "../../utils/segment"; +import { signer } from "../../utils/wallet"; +import { connect } from "../worker"; + +const kms = new KeyManagementServiceClient(); +const secrets = new SecretManagerServiceClient(); + +supervise( + name, + Promise.all([ + secret("redis-url", secrets).then((url) => connect(url)), + secret("poke-onesignal-api-key", secrets).then((key) => createOnesignal(key)), + signer("poker", kms), + secret("poke-segment-write-key", secrets).then((key) => createSegment(key)), + ]).then(([bullmq, onesignal, poker, segment]) => + own( + worker({ bullmq, onesignal, poker, segment }), + () => bullmq.quit(), + () => kms.close(), + () => secrets.close(), + () => segment.close(), + ), + ), +); diff --git a/server/workers/poke/job.ts b/server/workers/poke/job.ts new file mode 100644 index 0000000000..65ab56756d --- /dev/null +++ b/server/workers/poke/job.ts @@ -0,0 +1,16 @@ +import type { Address, Hex } from "@exactly/common/validation"; + +export const name = "poke"; +export const attempts = 10; + +export type Job = { + account: Address; + assets?: Address[]; + chainId: number; + factory: Address; + origin: "activity" | "allow"; + publicKey: Hex; + sentryBaggage?: string; + sentryTrace?: string; + source: null | string; +}; diff --git a/server/workers/poke/queue.ts b/server/workers/poke/queue.ts new file mode 100644 index 0000000000..6ba4e92a4b --- /dev/null +++ b/server/workers/poke/queue.ts @@ -0,0 +1,25 @@ +import { attempts, name, type Job } from "./job"; +import createQueue from "../queue"; + +import type { Redis } from "ioredis"; + +export default function queue(bullmq: Redis) { + const instance = createQueue(name, attempts, bullmq); + return { + close: () => instance.close(), + async enqueue({ + account, + assets, + chainId, + factory, + origin, + publicKey, + source, + }: Omit) { + await instance.enqueue( + { account, assets, chainId, factory, origin, publicKey, source }, + [chainId, account, ...(assets ?? [])].join("-"), + ); + }, + }; +} diff --git a/server/workers/poke/worker.ts b/server/workers/poke/worker.ts new file mode 100644 index 0000000000..74c30114dd --- /dev/null +++ b/server/workers/poke/worker.ts @@ -0,0 +1,174 @@ +import { captureException, withScope } from "@sentry/node"; +import { parse } from "valibot"; +import { bytesToBigInt, erc20Abi, hexToBytes, type LocalAccount } from "viem"; + +import exaChain, { + auditorAbi, + exaAccountFactoryAbi, + exaPluginAbi, + exaPreviewerAbi, + exaPreviewerAddress, + marketAbi, + upgradeableModularAccountAbi, + wethAddress, +} from "@exactly/common/generated/chain"; +import { Address } from "@exactly/common/validation"; + +import { attempts, name, type Job } from "./job"; +import t from "../../i18n"; +import { own } from "../../supervise"; +import { NETWORKS } from "../../utils/alchemy"; +import decodePublicKey from "../../utils/decodePublicKey"; +import publicClient from "../../utils/publicClient"; +import revertFingerprint from "../../utils/revertFingerprint"; +import createWallet from "../../utils/wallet"; +import createCredit from "../credit/queue"; +import createWorker from "../worker"; + +import type createOnesignal from "../../utils/onesignal"; +import type createSegment from "../../utils/segment"; +import type { Redis } from "ioredis"; + +export default function worker({ + bullmq, + onesignal, + poker, + segment, +}: { + bullmq: Redis; + onesignal: ReturnType; + poker: LocalAccount; + segment: ReturnType; +}) { + const credit = createCredit(bullmq, { count: 100 }); + return own( + createWorker({ + attempts, + bullmq, + failed(job, error) { + withScope((scope) => { + if (job) scope.setUser({ id: job.data.account }); + const noBalance = error.message === NO_BALANCE; + captureException(error, { + extra: { account: job?.data.account, attempts: job?.attemptsMade, id: job?.id }, + fingerprint: noBalance ? ["{{ default }}", "NoBalance"] : revertFingerprint(error), + level: noBalance ? "warning" : "error", + tags: { queue: name, job: job?.name }, + }); + }); + }, + name, + async process(job, span) { + const chain = [...NETWORKS.values()].find(({ id }) => id === job.data.chainId); + if (!chain) throw new Error(`unsupported chain ${job.data.chainId}`); + const wallet = createWallet(poker, chain); + const isDeployed = !!(await wallet.getCode({ address: job.data.account })); + span.setAttribute("exa.new", !isDeployed); + if (!isDeployed) { + await wallet.exaSend( + { name: "create account", op: "exa.account", attributes: { account: job.data.account } }, + { + address: job.data.factory, + functionName: "createAccount", + args: [0n, [decodePublicKey(new Uint8Array(hexToBytes(job.data.publicKey)), bytesToBigInt)]], + abi: exaAccountFactoryAbi, + }, + chain.id === exaChain.id ? {} : { fees: "auto" }, + ); + segment.track({ + event: "AccountFunded", + userId: job.data.account, + properties: { source: job.data.source }, + }); + } + if (chain.id === exaChain.id) { + const marketsByAsset = await publicClient + .readContract({ address: exaPreviewerAddress, functionName: "assets", abi: exaPreviewerAbi }) + .then( + (markets) => + new Map( + markets.map(({ asset, market }) => [parse(Address, asset), parse(Address, market)]), + ), + ); + const balances = await Promise.all( + [...new Set(job.data.assets ?? [ETH, ...marketsByAsset.keys()])] + .filter((asset) => asset === ETH || marketsByAsset.has(asset)) + .map(async (asset) => ({ + asset, + balance: + asset === ETH + ? await publicClient.getBalance({ address: job.data.account }) + : await publicClient.readContract({ + address: asset, + functionName: "balanceOf", + args: [job.data.account], + abi: erc20Abi, + }), + })), + ); + const hasETH = balances.some(({ asset, balance }) => asset === ETH && balance > 0n); + const pending: Address[] = []; + let poked = false; + for (const [index, { asset, balance }] of balances.entries()) { + if (hasETH && asset === WETH) continue; + if (balance === 0n) { + pending.push(asset); + continue; + } + const receipt = await wallet.exaSend( + { name: "poke account", op: "exa.poke", attributes: { account: job.data.account, asset } }, + asset === ETH + ? { address: job.data.account, abi: accountAbi, functionName: "pokeETH" } + : { + address: job.data.account, + abi: accountAbi, + functionName: "poke", + args: [marketsByAsset.get(asset)!], // eslint-disable-line @typescript-eslint/no-non-null-assertion + }, + { ignore: [NO_BALANCE] }, + ); + if (!receipt) { + pending.push(asset); + continue; + } + poked = true; + if (job.data.origin === "activity") { + await job.updateData({ + ...job.data, + assets: [ + ...pending, + ...balances + .slice(index + 1) + .map(({ asset: remaining }) => remaining) + .filter((remaining) => !(hasETH && remaining === WETH)), + ], + }); + } + } + if (job.data.origin === "activity" && pending.length > 0) { + await job.updateData({ ...job.data, assets: pending }); + throw new Error(NO_BALANCE); + } + if (job.data.origin === "allow" && poked) { + await onesignal + .sendPushNotification({ + userId: job.data.account, + headings: t("Account assets updated"), + contents: t("Your funds are ready to use"), + }) + .catch((error: unknown) => captureException(error, { level: "error" })); + } + if (job.data.origin === "activity") { + await credit.enqueue(job.data.account, `poke-${job.id}`); + } + } + }, + }), + () => credit.close(), + ); +} + +const ETH = parse(Address, "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); +const NO_BALANCE = "NoBalance()"; +const WETH = parse(Address, wethAddress); +const accountAbi = [...exaPluginAbi, ...upgradeableModularAccountAbi, ...auditorAbi, ...marketAbi]; From c34c4706742ccffe50d9cecc1e643e1f3e4fd0b0 Mon Sep 17 00:00:00 2001 From: danilo neves cruz Date: Wed, 22 Jul 2026 17:42:10 +0200 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9C=A8=20server:=20automate=20firewall?= =?UTF-8?q?=20allow=20after=20kyc=20approval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit co-authored-by: Miguel Diaz --- .changeset/blue-bottles-wave.md | 5 + .changeset/free-lamps-pump.md | 5 + infra/utils/modules.ts | 1 + server/hooks/persona.ts | 21 +- server/index.ts | 4 + server/test/e2e.ts | 2 + server/test/hooks/persona.test.ts | 120 ++++++++++- server/test/mocks/deployments.ts | 14 +- server/test/workers/allow.test.ts | 329 ++++++++++++++++++++++++++++++ server/test/workers/bin.test.ts | 40 +++- server/workers/allow/bin.ts | 25 +++ server/workers/allow/job.ts | 6 + server/workers/allow/queue.ts | 21 ++ server/workers/allow/worker.ts | 57 ++++++ 14 files changed, 641 insertions(+), 9 deletions(-) create mode 100644 .changeset/blue-bottles-wave.md create mode 100644 .changeset/free-lamps-pump.md create mode 100644 server/test/workers/allow.test.ts create mode 100644 server/workers/allow/bin.ts create mode 100644 server/workers/allow/job.ts create mode 100644 server/workers/allow/queue.ts create mode 100644 server/workers/allow/worker.ts diff --git a/.changeset/blue-bottles-wave.md b/.changeset/blue-bottles-wave.md new file mode 100644 index 0000000000..fce6e946b6 --- /dev/null +++ b/.changeset/blue-bottles-wave.md @@ -0,0 +1,5 @@ +--- +"@exactly/server": patch +--- + +✨ automate firewall allow after kyc approval diff --git a/.changeset/free-lamps-pump.md b/.changeset/free-lamps-pump.md new file mode 100644 index 0000000000..75acd1091f --- /dev/null +++ b/.changeset/free-lamps-pump.md @@ -0,0 +1,5 @@ +--- +"@exactly/infra": minor +--- + +🧱 setup allow queue worker diff --git a/infra/utils/modules.ts b/infra/utils/modules.ts index bd996dcf7a..5eb230d5ec 100644 --- a/infra/utils/modules.ts +++ b/infra/utils/modules.ts @@ -8,6 +8,7 @@ export default define({ }, }, workers: { + allow: { signers: ["allower"] }, chat: { env: { WHATSAPP_PHONE_NUMBER_ID: "whatsappPhoneNumberId" }, secrets: ["anthropic-api-key", "whatsapp-access-token"], diff --git a/server/hooks/persona.ts b/server/hooks/persona.ts index 9a427d83f7..e3cd3eb839 100644 --- a/server/hooks/persona.ts +++ b/server/hooks/persona.ts @@ -16,6 +16,7 @@ import { number, object, optional, + parse, picklist, pipe, safeParse, @@ -23,7 +24,9 @@ import { transform, union, } from "valibot"; +import { bytesToHex } from "viem"; +import chain, { firewallAddress } from "@exactly/common/generated/chain"; import { Address } from "@exactly/common/validation"; import { cards, credentials } from "../database/schema"; @@ -44,6 +47,7 @@ import type createPanda from "../utils/panda"; import type createPax from "../utils/pax"; import type createPersona from "../utils/persona"; import type createSardine from "../utils/sardine"; +import type createAllow from "../workers/allow/queue"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { InferOutput } from "valibot"; @@ -64,6 +68,7 @@ const Session = pipe( ); export default function hook({ + allow, database, panda, pax, @@ -71,6 +76,7 @@ export default function hook({ personaWebhookSecret, sardine, }: { + allow: ReturnType; database: NodePgDatabase; panda: ReturnType; pax: ReturnType; @@ -336,7 +342,7 @@ export default function hook({ const { referenceId, fields } = attributes; const credential = await database.query.credentials.findFirst({ - columns: { account: true, pandaId: true }, + columns: { account: true, factory: true, pandaId: true, publicKey: true, source: true }, where: eq(credentials.id, referenceId), }); if (!credential) { @@ -348,7 +354,18 @@ export default function hook({ getActiveSpan()?.setAttribute("exa.inquiryId", personaShareToken); const account = safeParse(Address, credential.account); + async function enqueueAllow(current: NonNullable) { + if (account.success && firewallAddress) + await allow.enqueue({ + account: account.output, + chainId: chain.id, + factory: parse(Address, current.factory), + publicKey: bytesToHex(current.publicKey), + source: current.source, + }); + } if (credential.pandaId) { + await enqueueAllow(credential); getActiveSpan()?.setAttribute(SEMANTIC_ATTRIBUTE_SENTRY_OP, "persona.inquiry.already-created"); return c.json({ code: "already created" }, 200); } @@ -407,6 +424,8 @@ export default function hook({ if (risk.level === "very_high") return c.json({ code: "very high risk" }, 200); } + await enqueueAllow(credential); + // TODO implement error handling to return 200 if event should not be retried const { id } = await panda.createUser({ accountPurpose: fields.accountPurpose.value, diff --git a/server/index.ts b/server/index.ts index a5becd8ba8..92f86b4af1 100644 --- a/server/index.ts +++ b/server/index.ts @@ -36,6 +36,7 @@ import createSardine from "./utils/sardine"; import createSegment from "./utils/segment"; import { legacy } from "./utils/wallet"; import createWalletExtension from "./utils/walletExtension"; +import createAllow from "./workers/allow/queue"; import createCredit from "./workers/credit/queue"; import createHook from "./workers/hook/queue"; import createHookWorker from "./workers/hook/worker"; @@ -69,6 +70,7 @@ const persona = createPersona( parse(pipe(string("persona key"), nonEmpty("persona key")), env.PERSONA_API_KEY), parse(pipe(string("persona url"), nonEmpty("persona url")), env.PERSONA_URL), ); +const allow = createAllow(redis); const credit = createCredit(redis); const poke = createPoke(redis); const refund = createRefund(redis); @@ -135,6 +137,7 @@ const pandaHook = createPandaHook({ webhook, }); const personaHook = createPersonaHook({ + allow, database, panda, pax, @@ -419,6 +422,7 @@ export const close = supervise( reminders().catch(reminders), ]), }, + () => allow.close(), closeMaturity, () => credit.close(), () => hookWorker.close(), diff --git a/server/test/e2e.ts b/server/test/e2e.ts index 748761fa1c..63ddf99998 100644 --- a/server/test/e2e.ts +++ b/server/test/e2e.ts @@ -21,6 +21,7 @@ import createOnesignal from "../utils/onesignal"; import createPanda from "../utils/panda"; import createSardine from "../utils/sardine"; import createSegment from "../utils/segment"; +import allowWorker from "../workers/allow/worker"; import creditWorker from "../workers/credit/worker"; import hookWorker from "../workers/hook/worker"; import pokeWorker from "../workers/poke/worker"; @@ -45,6 +46,7 @@ describe("e2e", () => { const sardine = createSardine("sardine", "https://sardine.test"); const segment = createSegment("segment"); const workers = [ + allowWorker({ allower: privateKeyToAccount(padHex("0xa11")), bullmq }), creditWorker({ bullmq, database, onesignal }), pokeWorker({ bullmq, onesignal, poker: privateKeyToAccount(padHex("0xb0b")), segment }), refundWorker({ diff --git a/server/test/hooks/persona.test.ts b/server/test/hooks/persona.test.ts index 9a707a271e..dd66a22b28 100644 --- a/server/test/hooks/persona.test.ts +++ b/server/test/hooks/persona.test.ts @@ -1,4 +1,4 @@ -import "../mocks/deployments"; +import deployments from "../mocks/deployments"; import "../mocks/panda"; import * as pax from "../mocks/pax"; import "../mocks/persona"; @@ -13,6 +13,7 @@ import { privateKeyToAddress } from "viem/accounts"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, inject, it, vi } from "vitest"; import deriveAddress from "@exactly/common/deriveAddress"; +import chain from "@exactly/common/generated/chain"; import database, { cards, credentials } from "../../database"; import createPersonaHook from "../../hooks/persona"; @@ -21,6 +22,12 @@ import createPax from "../../utils/pax"; import createPersona, * as Persona from "../../utils/persona"; import createSardine from "../../utils/sardine"; +import type createAllow from "../../workers/allow/queue"; + +const allow = { + close: vi.fn["close"]>(), + enqueue: vi.fn["enqueue"]>(), +}; const pandaConfig = { key: "panda", url: "https://panda.test" }; const paxConfig = { associateKey: "pax", key: "pax", url: "https://pax.test" }; const personaConfig = { key: "persona", url: "https://persona.test" }; @@ -28,6 +35,7 @@ const panda = createPanda(pandaConfig); const persona = Object.assign(createPersona(personaConfig.key, personaConfig.url), Persona); const sardineConfig = { key: "sardine", url: "https://api.sardine.ai" }; const personaHook = createPersonaHook({ + allow, database, panda, pax: createPax(paxConfig), @@ -40,6 +48,11 @@ const appClient = testClient(app); vi.mock("@sentry/node", { spy: true }); +beforeEach(() => { + deployments.setFirewall(inject("Firewall")); + allow.close.mockReset().mockResolvedValue(); + allow.enqueue.mockReset().mockResolvedValue(); +}); describe("with reference", () => { const referenceId = "hook-persona"; const owner = privateKeyToAddress(padHex("0x123")); @@ -92,6 +105,13 @@ describe("with reference", () => { expect(p?.pandaId).toBe("pandaId"); + expect(allow.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + chainId: chain.id, + factory, + publicKey: owner.toLowerCase(), + source: null, + }); expect(response.status).toBe(200); }); @@ -218,6 +238,13 @@ describe("with reference", () => { }); expect(response.status).toBe(200); + expect(allow.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + chainId: chain.id, + factory, + publicKey: owner.toLowerCase(), + source: null, + }); expect(panda.createUser).not.toHaveBeenCalled(); }); @@ -405,15 +432,16 @@ describe("with reference", () => { }); describe("persona hook", () => { + const account = deriveAddress(inject("ExaAccountFactory"), { + x: padHex(privateKeyToAddress(padHex("0x420"))), + y: zeroHash, + }); beforeAll(async () => { await database.insert(credentials).values({ id: "persona-ref", publicKey: new Uint8Array(), factory: inject("ExaAccountFactory"), - account: deriveAddress(inject("ExaAccountFactory"), { - x: padHex(privateKeyToAddress(padHex("0x420"))), - y: zeroHash, - }), + account, pandaId: null, }); }); @@ -427,7 +455,8 @@ describe("persona hook", () => { afterEach(async () => { await new Promise((resolve) => setImmediate(resolve)); - await database.update(credentials).set({ pandaId: null }).where(eq(credentials.id, "persona-ref")); + await database.update(credentials).set({ account, pandaId: null }).where(eq(credentials.id, "persona-ref")); + vi.clearAllMocks(); vi.restoreAllMocks(); }); @@ -452,6 +481,13 @@ describe("persona hook", () => { }); expect(response.status).toBe(200); + expect(allow.enqueue).toHaveBeenCalledExactlyOnceWith({ + account, + chainId: chain.id, + factory: inject("ExaAccountFactory"), + publicKey: "0x", + source: null, + }); expect(panda.createUser).toHaveBeenCalledWith({ accountPurpose: "business", annualSalary: "100000", @@ -503,6 +539,57 @@ describe("persona hook", () => { expect(response.status).toBe(200); await expect(response.json()).resolves.toStrictEqual({ code: "very high risk" }); + expect(allow.enqueue).not.toHaveBeenCalled(); + expect(panda.createUser).not.toHaveBeenCalled(); + }); + + it("skips allow for invalid credential accounts", async () => { + await database.update(credentials).set({ account: "invalid" }).where(eq(credentials.id, "persona-ref")); + + const response = await postInquiry(); + + expect(response.status).toBe(200); + expect(allow.enqueue).not.toHaveBeenCalled(); + expect(panda.createUser).toHaveBeenCalledOnce(); + }); + + it("skips allow without a firewall deployment", async () => { + deployments.setFirewall(undefined); + + const response = await postInquiry(); + + expect(response.status).toBe(200); + expect(allow.enqueue).not.toHaveBeenCalled(); + expect(panda.createUser).toHaveBeenCalledOnce(); + }); + + it("fails before panda creation when allow cannot be queued", async () => { + const error = new Error("redis unavailable"); + const errorConsole = vi.spyOn(console, "error").mockImplementation(() => undefined); + allow.enqueue.mockRejectedValueOnce(error); + + const response = await appClient.index.$post({ + header: { + "persona-signature": "t=1733865120,v1=debbacfe1b0c5f8797a1d68e8428fba435aa4ca3b5d9a328c3c96ee4d04d84df", + }, + json: { + ...validPayload, + data: { + ...validPayload.data, + attributes: { + ...validPayload.data.attributes, + payload: { + ...validPayload.data.attributes.payload, + included: [...validPayload.data.attributes.payload.included], + }, + }, + }, + }, + }); + + expect(response.status).toBe(500); + expect(errorConsole).toHaveBeenCalledWith(error); + expect(allow.enqueue).toHaveBeenCalledOnce(); expect(panda.createUser).not.toHaveBeenCalled(); }); @@ -1885,3 +1972,24 @@ const mantecaPayload = { }, }, } as const; + +function postInquiry() { + return appClient.index.$post({ + header: { + "persona-signature": "t=1733865120,v1=debbacfe1b0c5f8797a1d68e8428fba435aa4ca3b5d9a328c3c96ee4d04d84df", + }, + json: { + ...validPayload, + data: { + ...validPayload.data, + attributes: { + ...validPayload.data.attributes, + payload: { + ...validPayload.data.attributes.payload, + included: [...validPayload.data.attributes.payload.included], + }, + }, + }, + }, + }); +} diff --git a/server/test/mocks/deployments.ts b/server/test/mocks/deployments.ts index 10690aa529..6c99cae1e2 100644 --- a/server/test/mocks/deployments.ts +++ b/server/test/mocks/deployments.ts @@ -1,6 +1,16 @@ import { foundry } from "viem/chains"; import { inject, vi } from "vitest"; +import type { Address } from "@exactly/common/validation"; + +let firewall: Address | undefined = inject("Firewall"); + +export default { + setFirewall(address: Address | undefined) { + firewall = address; + }, +}; + vi.mock("@exactly/common/generated/chain", async (importOriginal) => ({ ...(await importOriginal()), default: { ...foundry, rpcUrls: { ...foundry.rpcUrls, alchemy: foundry.rpcUrls.default } }, @@ -9,7 +19,9 @@ vi.mock("@exactly/common/generated/chain", async (importOriginal) => ({ exaAccountFactoryAddress: inject("ExaAccountFactory"), exaPluginAddress: inject("ExaPlugin"), exaPreviewerAddress: inject("ExaPreviewer"), - firewallAddress: inject("Firewall"), + get firewallAddress() { + return firewall; + }, issuerCheckerAddress: inject("IssuerChecker"), marketUSDCAddress: inject("MarketUSDC"), marketWETHAddress: inject("MarketWETH"), diff --git a/server/test/workers/allow.test.ts b/server/test/workers/allow.test.ts new file mode 100644 index 0000000000..9a1f9d3772 --- /dev/null +++ b/server/test/workers/allow.test.ts @@ -0,0 +1,329 @@ +import "../mocks/sentry"; + +import { captureException, continueTrace, startSpan, withScope } from "@sentry/node"; +import { Queue, QueueEvents, type Job, type JobsOptions } from "bullmq"; +import { parse } from "valibot"; +import { padHex } from "viem"; +import { privateKeyToAccount } from "viem/accounts"; +import { afterAll, afterEach, beforeEach, describe, expect, inject, it, vi } from "vitest"; + +import chain, { firewallAbi } from "@exactly/common/generated/chain"; +import { Address } from "@exactly/common/validation"; + +import { bullmq } from "../../utils/redis"; +import createAllow from "../../workers/allow/queue"; +import allowWorker from "../../workers/allow/worker"; + +import type { Job as Allow } from "../../workers/allow/job"; +import type createPoke from "../../workers/poke/queue"; +import type * as C from "@exactly/common/generated/chain"; +import type * as sentry from "@sentry/node"; + +const factory = inject("ExaAccountFactory"); +const account = parse(Address, padHex("0xb0b", { size: 20 })); +const firewall = inject("Firewall"); +const request = { account, chainId: chain.id, factory, publicKey: "0x1234" as const, source: null }; +const allower = privateKeyToAccount(padHex("0xa11")); +const allow = createAllow(bullmq); +const mocks = vi.hoisted(() => ({ + closePoke: vi.fn["close"]>(), + createPoke: vi.fn(), + enqueuePoke: vi.fn["enqueue"]>(), + exaSend: vi.fn(), + firewall: vi.fn<() => Address | undefined>(), + wallet: vi.fn(), +})); + +vi.mock("../../utils/wallet", () => ({ default: mocks.wallet })); +vi.mock("../../workers/poke/queue", () => ({ default: mocks.createPoke })); + +vi.mock("@exactly/common/generated/chain", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + get firewallAddress() { + return mocks.firewall(); + }, + }; +}); + +const queue = new Queue("allow", { connection: bullmq }); +const events = new QueueEvents("allow", { connection: bullmq }); +let worker: Awaited>; + +async function jobFinished( + current: Address, + options?: JobsOptions, + trace?: Pick, +) { + const job = await queue.add( + "allow", + { ...request, account: current, ...trace }, + { attempts: 1, jobId: current, removeOnComplete: true, removeOnFail: true, ...options }, + ); + await job.waitUntilFinished(events).catch(async (error: unknown) => { + await vi.waitUntil(() => vi.mocked(captureException).mock.calls.length > 0); + throw error; + }); + return job; +} + +async function spyScopeSetUser() { + const { withScope: realWithScope } = await vi.importActual("@sentry/node"); + const setUser = vi.fn(); + vi.mocked(withScope).mockImplementation((_scopeOrCallback, _callback?) => + realWithScope((scope) => { + const originalSetUser = scope.setUser.bind(scope); + scope.setUser = (...args: Parameters) => { + setUser(...args); + return originalSetUser(...args); + }; + return ((_callback ?? _scopeOrCallback) as NonNullable)(scope); + }), + ); + return setUser; +} + +afterAll(async () => { + await Promise.all([events.close(), queue.close(), allow.close()]); +}); + +beforeEach(async () => { + vi.restoreAllMocks(); + mocks.closePoke.mockReset().mockResolvedValue(); + mocks.createPoke.mockReset().mockReturnValue({ close: mocks.closePoke, enqueue: mocks.enqueuePoke }); + mocks.enqueuePoke.mockReset().mockResolvedValue(); + mocks.exaSend.mockReset().mockResolvedValue({}); + mocks.firewall.mockReset().mockReturnValue(firewall); + mocks.wallet.mockReset().mockReturnValue({ exaSend: mocks.exaSend }); + vi.clearAllMocks(); + await queue.drain(true); + await queue.clean(0, 1000, "completed"); + await queue.clean(0, 1000, "failed"); +}); + +describe("allow queue", () => { + it("publishes firewall allow jobs", async () => { + await expect(allow.enqueue(request)).resolves.toBeUndefined(); + + const job = await queue.getJob(account); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(account); + expect(job.name).toBe("allow"); + expect(job.data).toStrictEqual({ + ...request, + sentryBaggage: expect.any(String) as string, + sentryTrace: expect.any(String) as string, + }); + expect(job.opts).toStrictEqual({ + attempts: 10, + backoff: { type: "exponential", delay: 1000 }, + jobId: account, + removeOnComplete: true, + removeOnFail: { count: 1000, age: 7 * 24 * 3600 }, + }); + await expect(job.getState()).resolves.toBe("waiting"); + expect(startSpan).toHaveBeenCalledWith( + { name: "allow", op: "queue.publish", attributes: { "messaging.destination.name": "allow" } }, + expect.any(Function), + ); + expect(captureException).not.toHaveBeenCalled(); + await job.remove(); + }); + + it("propagates queue failures", async () => { + const error = new Error("queue error"); + vi.spyOn(Queue.prototype, "add").mockRejectedValueOnce(error); + + await expect(allow.enqueue(request)).rejects.toThrow(error); + + expect(captureException).not.toHaveBeenCalled(); + }); +}); + +describe("allow worker", () => { + beforeEach(async () => { + worker = allowWorker({ allower, bullmq }); + await worker.ready; + }); + + afterEach(async () => { + await worker.close(); + }); + + it("allows queued accounts with the isolated wallet", async () => { + await jobFinished(account); + + expect(mocks.wallet).toHaveBeenCalledExactlyOnceWith(allower); + expect(mocks.exaSend).toHaveBeenCalledExactlyOnceWith( + { name: "firewall.allow", op: "exa.firewall", attributes: { account } }, + { address: firewall, functionName: "allow", args: [account, true], abi: firewallAbi }, + { ignore: [`AlreadyAllowed(${account})`] }, + ); + expect(mocks.enqueuePoke).toHaveBeenCalledExactlyOnceWith({ ...request, origin: "allow" }); + expect(vi.mocked(startSpan)).toHaveBeenCalledWith( + expect.objectContaining({ forceTransaction: true, name: "allow worker" }), + expect.any(Function), + ); + expect(vi.mocked(startSpan)).toHaveBeenCalledWith( + expect.objectContaining({ name: "allow", op: "queue.process" }), + expect.any(Function), + ); + expect(vi.mocked(captureException)).not.toHaveBeenCalled(); + }); + + it("queues the poke only after allow settles", async () => { + const deferred = Promise.withResolvers(); + mocks.exaSend.mockReturnValueOnce(deferred.promise); + + const processing = jobFinished(account); + await vi.waitUntil(() => mocks.exaSend.mock.calls.length === 1); + expect(mocks.enqueuePoke).not.toHaveBeenCalled(); + deferred.resolve({}); + await processing; + + expect(mocks.enqueuePoke).toHaveBeenCalledOnce(); + }); + + it("closes the poke queue after active jobs settle", async () => { + const deferred = Promise.withResolvers(); + mocks.exaSend.mockReturnValueOnce(deferred.promise); + + const processing = jobFinished(account); + await vi.waitUntil(() => mocks.exaSend.mock.calls.length === 1); + const closeWorker = vi.spyOn(worker.queue, "close"); + const closing = worker.close(); + expect(worker.close()).toBe(closing); + await vi.waitUntil(() => closeWorker.mock.calls.length === 1); + + expect(mocks.closePoke).not.toHaveBeenCalled(); + + deferred.resolve({}); + await processing; + await closing; + expect(mocks.closePoke).toHaveBeenCalledExactlyOnceWith(); + }); + + it("retries allow failures", async () => { + mocks.exaSend.mockRejectedValueOnce(new Error("rpc unavailable")).mockResolvedValueOnce({}); + + await jobFinished(account, { attempts: 2, backoff: { type: "fixed", delay: 1 } }); + + expect(mocks.exaSend).toHaveBeenCalledTimes(2); + expect(mocks.enqueuePoke).toHaveBeenCalledExactlyOnceWith({ ...request, origin: "allow" }); + expect(vi.mocked(captureException)).not.toHaveBeenCalled(); + }); + + it("captures terminal failures", async () => { + const error = new Error("allow failed"); + const setUser = await spyScopeSetUser(); + mocks.exaSend.mockRejectedValue(error); + + await expect( + jobFinished(account, { attempts: 2, backoff: { type: "fixed", delay: 1 }, removeOnFail: false }), + ).rejects.toThrow("allow failed"); + + await expect(queue.getFailedCount()).resolves.toBe(1); + await expect(queue.getJobState(account)).resolves.toBe("failed"); + const [job] = await queue.getFailed(); + if (!job) throw new Error("job not found"); + expect(job.id).toBe(account); + expect(job.failedReason).toBe("allow failed"); + expect(job.attemptsMade).toBe(2); + expect(job.stacktrace).toHaveLength(2); + expect(mocks.exaSend).toHaveBeenCalledTimes(2); + expect(mocks.enqueuePoke).not.toHaveBeenCalled(); + expect(setUser).toHaveBeenCalledExactlyOnceWith({ id: account }); + expect(vi.mocked(captureException)).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account, attempts: 2, id: account }, + level: "error", + tags: { queue: "allow", job: "allow" }, + }); + await job.remove(); + }); + + it("fails when the firewall is unavailable", async () => { + const setUser = await spyScopeSetUser(); + mocks.firewall.mockReset(); + + await expect(jobFinished(account)).rejects.toThrow(); + + expect(mocks.exaSend).not.toHaveBeenCalled(); + expect(mocks.enqueuePoke).not.toHaveBeenCalled(); + expect(setUser).toHaveBeenCalledExactlyOnceWith({ id: account }); + const captured = vi.mocked(captureException).mock.calls[0]?.[0]; + if (!(captured instanceof Error)) throw new Error("missing captured error"); + expect(captured.message).toBe("bad address"); + expect(vi.mocked(captureException)).toHaveBeenCalledExactlyOnceWith(captured, { + extra: { account, attempts: 1, id: account }, + level: "error", + tags: { queue: "allow", job: "allow" }, + }); + }); + + it("retries poke publication failures", async () => { + const error = new Error("poke unavailable"); + const setUser = await spyScopeSetUser(); + mocks.enqueuePoke.mockRejectedValue(error); + + await expect(jobFinished(account, { attempts: 2, backoff: { type: "fixed", delay: 1 } })).rejects.toThrow( + "poke unavailable", + ); + + expect(mocks.exaSend).toHaveBeenCalledTimes(2); + expect(mocks.enqueuePoke).toHaveBeenCalledTimes(2); + expect(setUser).toHaveBeenCalledExactlyOnceWith({ id: account }); + expect(vi.mocked(captureException)).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account, attempts: 2, id: account }, + level: "error", + tags: { queue: "allow", job: "allow" }, + }); + }); + + it("continues sentry traces", async () => { + await jobFinished(account, undefined, { sentryBaggage: "baggage", sentryTrace: "trace" }); + + expect(vi.mocked(continueTrace)).toHaveBeenCalledWith( + { sentryTrace: "trace", baggage: "baggage" }, + expect.any(Function), + ); + }); + + it("captures worker errors", () => { + const error = new Error("worker error"); + + worker.queue.emit("error", error); + + expect(vi.mocked(captureException)).toHaveBeenCalledExactlyOnceWith(error, { + level: "error", + tags: { queue: "allow" }, + }); + }); + + it("captures failed events without a job", async () => { + const error = new Error("failed event error"); + const setUser = await spyScopeSetUser(); + + worker.queue.emit("failed", undefined, error, "active"); + + expect(setUser).not.toHaveBeenCalled(); + expect(vi.mocked(captureException)).toHaveBeenCalledExactlyOnceWith(error, { + extra: { account: undefined, attempts: undefined, id: undefined }, + level: "error", + tags: { queue: "allow", job: undefined }, + }); + }); + + it("skips intermediate failed events with default attempts", () => { + const error = new Error("failed event error"); + + worker.queue.emit( + "failed", + { attemptsMade: 9, data: { account }, name: "allow", opts: {} } as Job, + error, + "active", + ); + + expect(vi.mocked(captureException)).not.toHaveBeenCalled(); + }); +}); diff --git a/server/test/workers/bin.test.ts b/server/test/workers/bin.test.ts index e67293c8f1..177506d4c6 100644 --- a/server/test/workers/bin.test.ts +++ b/server/test/workers/bin.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type * as Supervise from "../../supervise"; const alchemy = {}; +const allower = privateKeyToAccount(padHex("0xa11")); const refunder = privateKeyToAccount(padHex("0xfee")); const database = { $client: { end: vi.fn<() => Promise>() } }; const onesignal = {}; @@ -15,6 +16,7 @@ const segment = { close: vi.fn<() => Promise>() }; const whatsapp = {}; const mocks = { alchemy: vi.fn<(key: string) => object>(), + allow: vi.fn<(config: { allower: typeof allower; bullmq: object }) => Handle>(), chat: vi.fn<(config: { anthropicKey: string; bullmq: object; whatsapp: object }) => Handle>(), close: vi.fn<() => Promise>(), credit: vi.fn<(config: { bullmq: object; database: typeof database; onesignal: object }) => Handle>(), @@ -53,6 +55,7 @@ afterEach(() => { beforeEach(() => { vi.resetModules(); mocks.alchemy.mockReset().mockReturnValue(alchemy); + mocks.allow.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.close.mockReset().mockResolvedValue(); mocks.credit.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.drizzle.mockReset().mockReturnValue(database); @@ -65,7 +68,9 @@ beforeEach(() => { mocks.sardine.mockReset().mockReturnValue(sardine); mocks.secret.mockReset().mockImplementation((name) => Promise.resolve(name)); mocks.segment.mockReset().mockReturnValue(segment); - mocks.signer.mockReset().mockImplementation((name) => Promise.resolve(name === "poker" ? poker : refunder)); + mocks.signer + .mockReset() + .mockImplementation((name) => Promise.resolve(name === "allower" ? allower : name === "poker" ? poker : refunder)); mocks.subscribe.mockReset().mockReturnValue({ close: mocks.close, ready: Promise.resolve() }); mocks.supervise.mockReset(); mocks.whatsapp.mockReset().mockReturnValue(whatsapp); @@ -90,6 +95,7 @@ beforeEach(() => { vi.doMock("../../utils/segment", () => ({ default: mocks.segment })); vi.doMock("../../utils/wallet", () => ({ signer: mocks.signer })); vi.doMock("../../utils/whatsapp", () => ({ default: mocks.whatsapp })); + vi.doMock("../../workers/allow/worker", () => ({ default: mocks.allow })); vi.doMock("../../workers/chat/worker", () => ({ default: mocks.chat })); vi.doMock("../../workers/credit/worker", () => ({ default: mocks.credit })); vi.doMock("../../workers/hook/worker", () => ({ default: mocks.hook })); @@ -99,6 +105,38 @@ beforeEach(() => { }); describe("bin", () => { + it("resolves allow private config before constructing and supervising its worker", async () => { + await import("../../workers/allow/bin"); + + const created = mocks.supervise.mock.calls[0]?.[1]; + if (!created) throw new Error("missing worker"); + expect(mocks.supervise).toHaveBeenCalledExactlyOnceWith("allow", created); + await created; + expect(mocks.secret.mock.calls.map(([secret]) => secret)).toStrictEqual(["redis-url"]); + expect(new Set(mocks.secret.mock.calls.map(([, secrets]) => secrets)).size).toBe(1); + expect(mocks.signer.mock.calls.map(([signer]) => signer)).toStrictEqual(["allower"]); + expect(mocks.allow).toHaveBeenCalledExactlyOnceWith({ + allower, + bullmq: expect.objectContaining({ redisUrl: "redis-url", options: { maxRetriesPerRequest: null } }) as object, + }); + }); + + it("fails before constructing the allow worker without its allower account", async () => { + const error = new Error("missing allower"); + mocks.signer.mockRejectedValueOnce(error); + mocks.supervise.mockImplementation((_, created) => { + created.catch(() => undefined); + }); + + await import("../../workers/allow/bin"); + const created = mocks.supervise.mock.calls[0]?.[1]; + if (!created) throw new Error("missing worker"); + + await expect(created).rejects.toBe(error); + expect(mocks.signer.mock.calls.map(([signer]) => signer)).toStrictEqual(["allower"]); + expect(mocks.allow).not.toHaveBeenCalled(); + }); + it("resolves chat config before constructing and supervising its worker", async () => { await import("../../workers/chat/bin"); diff --git a/server/workers/allow/bin.ts b/server/workers/allow/bin.ts new file mode 100644 index 0000000000..ac80d4f429 --- /dev/null +++ b/server/workers/allow/bin.ts @@ -0,0 +1,25 @@ +import { KeyManagementServiceClient } from "@google-cloud/kms"; +import { SecretManagerServiceClient } from "@google-cloud/secret-manager"; + +import { name } from "./job"; +import worker from "./worker"; +import supervise, { own } from "../../supervise"; +import secret from "../../utils/secret"; +import { signer } from "../../utils/wallet"; +import { connect } from "../worker"; + +const kms = new KeyManagementServiceClient(); +const secrets = new SecretManagerServiceClient(); + +supervise( + name, + Promise.all([secret("redis-url", secrets).then((url) => connect(url)), signer("allower", kms)]).then( + ([bullmq, allower]) => + own( + worker({ allower, bullmq }), + () => bullmq.quit(), + () => kms.close(), + () => secrets.close(), + ), + ), +); diff --git a/server/workers/allow/job.ts b/server/workers/allow/job.ts new file mode 100644 index 0000000000..13d61bfd1a --- /dev/null +++ b/server/workers/allow/job.ts @@ -0,0 +1,6 @@ +import type { Job as Poke } from "../poke/job"; + +export const name = "allow"; +export const attempts = 10; + +export type Job = Omit; diff --git a/server/workers/allow/queue.ts b/server/workers/allow/queue.ts new file mode 100644 index 0000000000..20dbfc3f51 --- /dev/null +++ b/server/workers/allow/queue.ts @@ -0,0 +1,21 @@ +import { attempts, name, type Job } from "./job"; +import createQueue from "../queue"; + +import type { Redis } from "ioredis"; + +export default function queue(redis: Redis) { + const instance = createQueue(name, attempts, redis); + return { + close: () => instance.close(), + async enqueue({ + account, + assets, + chainId, + factory, + publicKey, + source, + }: Omit) { + await instance.enqueue({ account, assets, chainId, factory, publicKey, source }, account); + }, + }; +} diff --git a/server/workers/allow/worker.ts b/server/workers/allow/worker.ts new file mode 100644 index 0000000000..e821eecf11 --- /dev/null +++ b/server/workers/allow/worker.ts @@ -0,0 +1,57 @@ +import { captureException, withScope } from "@sentry/node"; +import { parse } from "valibot"; + +import { firewallAbi, firewallAddress } from "@exactly/common/generated/chain"; +import { Address } from "@exactly/common/validation"; + +import { attempts, name, type Job } from "./job"; +import { own } from "../../supervise"; +import wallet from "../../utils/wallet"; +import createPoke from "../poke/queue"; +import createWorker from "../worker"; + +import type { Redis } from "ioredis"; +import type { LocalAccount } from "viem"; + +export default function worker({ allower, bullmq }: { allower: LocalAccount; bullmq: Redis }) { + const poke = createPoke(bullmq); + return own( + createWorker({ + attempts, + bullmq, + failed(job, error) { + withScope((scope) => { + if (job) scope.setUser({ id: job.data.account }); + captureException(error, { + extra: { account: job?.data.account, attempts: job?.attemptsMade, id: job?.id }, + level: "error", + tags: { queue: name, job: job?.name }, + }); + }); + }, + name, + async process(job) { + await wallet(allower).exaSend( + { name: "firewall.allow", op: "exa.firewall", attributes: { account: job.data.account } }, + { + address: parse(Address, firewallAddress), + functionName: "allow", + args: [job.data.account, true], + abi: firewallAbi, + }, + { ignore: [`AlreadyAllowed(${job.data.account})`] }, + ); + await poke.enqueue({ + account: job.data.account, + assets: job.data.assets, + chainId: job.data.chainId, + factory: job.data.factory, + origin: "allow", + publicKey: job.data.publicKey, + source: job.data.source, + }); + }, + }), + () => poke.close(), + ); +}