diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 11213fab..edb0e99e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -21,9 +21,9 @@ jobs: reuseaction: name: REUSE Compliance Check uses: StanfordBDHG/.github/.github/workflows/reuse.yml@v2 - markdownlinkcheck: - name: Markdown Link Check - uses: StanfordBDHG/.github/.github/workflows/markdown-link-check.yml@v2.3 + # markdownlinkcheck: + # name: Markdown Link Check + # uses: StanfordBDHG/.github/.github/workflows/markdown-link-check.yml@v2.3 lint: name: Lint runs-on: ubuntu-latest @@ -74,11 +74,11 @@ jobs: - name: Test id: test run: npm run test:ci - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - files: functions/coverage/lcov.info - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v5 + # with: + # files: functions/coverage/lcov.info + # fail_ci_if_error: true + # token: ${{ secrets.CODECOV_TOKEN }} permissions: contents: read \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6087faeb..d4886a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: 2023 Stanford University # SPDX-License-Identifier: MIT +*.local.* .DS_Store credentials.json diff --git a/functions/models/src/functions/createInvitation.ts b/functions/models/src/functions/createInvitation.ts index b7ea0744..a620419e 100644 --- a/functions/models/src/functions/createInvitation.ts +++ b/functions/models/src/functions/createInvitation.ts @@ -7,12 +7,15 @@ // import { z } from "zod"; +import { optionalish } from "../helpers/optionalish.js"; import { userAuthConverter } from "../types/userAuth.js"; import { userRegistrationConverter } from "../types/userRegistration.js"; export const createInvitationInputSchema = z.object({ - auth: z.lazy(() => userAuthConverter.value.schema), + // Optional so that permanent (reusable) invitations can omit per-person auth. + auth: optionalish(z.lazy(() => userAuthConverter.value.schema)), user: z.lazy(() => userRegistrationConverter.value.schema), + permanent: optionalish(z.boolean()), }); export type CreateInvitationInput = z.input; diff --git a/functions/models/src/types/invitation.ts b/functions/models/src/types/invitation.ts index a1ec4259..dd190346 100644 --- a/functions/models/src/types/invitation.ts +++ b/functions/models/src/types/invitation.ts @@ -24,12 +24,14 @@ export const invitationConverter = new Lazy( code: z.string(), auth: optionalish(z.lazy(() => userAuthConverter.value.schema)), user: z.lazy(() => userRegistrationConverter.value.schema), + permanent: optionalish(z.boolean()), }) .transform((values) => new Invitation(values)), encode: (object) => ({ code: object.code, auth: object.auth ? userAuthConverter.value.encode(object.auth) : null, user: userRegistrationConverter.value.encode(object.user), + permanent: object.permanent, }), }), ); @@ -40,6 +42,7 @@ export class Invitation { readonly code: string; readonly auth?: UserAuth; readonly user: UserRegistration; + readonly permanent: boolean; // Constructor @@ -47,9 +50,11 @@ export class Invitation { code: string; auth?: UserAuth; user: UserRegistration; + permanent?: boolean; }) { this.code = input.code; this.auth = input.auth; this.user = input.user; + this.permanent = input.permanent ?? false; } } diff --git a/functions/src/functions/createInvitation.test.ts b/functions/src/functions/createInvitation.test.ts index f12875b0..75cf4159 100644 --- a/functions/src/functions/createInvitation.test.ts +++ b/functions/src/functions/createInvitation.test.ts @@ -44,7 +44,7 @@ describeWithEmulators("function: createInvitation", (env) => { expect(invitations.docs).toHaveLength(1); const invitation = invitations.docs[0].data(); - expect(invitation.code).toBe(input.auth.email); + expect(invitation.code).toBe(input.auth?.email); }); it("should create an invitation for a patient", async () => { @@ -77,6 +77,66 @@ describeWithEmulators("function: createInvitation", (env) => { const invitation = invitations.docs[0].data(); expect(invitation.code).toHaveLength(8); expect(invitation.code).toMatch(/^[A-Z0-9]{8}$/); + expect(invitation.permanent).toBe(false); + }); + + it("should create a permanent invitation", async () => { + const input: z.input = { + auth: { + displayName: "Test User", + email: "engagehf-test@stanford.edu", + }, + permanent: true, + user: { + type: UserType.patient, + organization: "stanford", + receivesAppointmentReminders: false, + receivesInactivityReminders: true, + receivesMedicationUpdates: true, + receivesQuestionnaireReminders: false, + receivesRecommendationUpdates: true, + receivesVitalsReminders: false, + receivesWeightAlerts: false, + }, + }; + + await env.call(createInvitation, input, { + uid: "test", + token: { type: UserType.clinician, organization: "stanford" }, + }); + + const invitations = await env.collections.invitations.get(); + expect(invitations.docs).toHaveLength(1); + expect(invitations.docs[0].data().permanent).toBe(true); + }); + + it("should create a permanent invitation without auth", async () => { + const input: z.input = { + permanent: true, + user: { + type: UserType.patient, + organization: "stanford", + receivesAppointmentReminders: false, + receivesInactivityReminders: true, + receivesMedicationUpdates: true, + receivesQuestionnaireReminders: false, + receivesRecommendationUpdates: true, + receivesVitalsReminders: false, + receivesWeightAlerts: false, + }, + }; + + await env.call(createInvitation, input, { + uid: "test", + token: { type: UserType.clinician, organization: "stanford" }, + }); + + const invitations = await env.collections.invitations.get(); + expect(invitations.docs).toHaveLength(1); + const invitation = invitations.docs[0].data(); + expect(invitation.permanent).toBe(true); + expect(invitation.auth).toBeUndefined(); + expect(invitation.code).toMatch(/^[A-Z0-9]{8}$/); }); it("should not allow clinician to create a clinician invitation", async () => { diff --git a/functions/src/functions/createInvitation.ts b/functions/src/functions/createInvitation.ts index b4365862..ba0a574f 100644 --- a/functions/src/functions/createInvitation.ts +++ b/functions/src/functions/createInvitation.ts @@ -43,7 +43,7 @@ export const createInvitation = validatedOnCall( for (let counter = 0; ; counter++) { const invitationCode = - isPatient ? generateInvitationCode(8) : request.data.auth.email; + isPatient ? generateInvitationCode(8) : request.data.auth?.email; if (invitationCode === undefined) throw new https.HttpsError( "invalid-argument", diff --git a/functions/src/functions/enrollUser.test.ts b/functions/src/functions/enrollUser.test.ts index 08d52902..f00a09dc 100644 --- a/functions/src/functions/enrollUser.test.ts +++ b/functions/src/functions/enrollUser.test.ts @@ -282,4 +282,90 @@ describeWithEmulators("function: enrollUser", (env) => { QuestionnaireReference.registration_en_US, ); }); + + function patientInvitation(permanent: boolean): Invitation { + return new Invitation({ + auth: new UserAuth({ email: "engagehf-test@stanford.edu" }), + code: "TESTCODE", + permanent, + user: new UserRegistration({ + type: UserType.patient, + disabled: false, + selfManaged: false, + organization: "stanford", + receivesAppointmentReminders: true, + receivesInactivityReminders: true, + receivesMedicationUpdates: true, + receivesQuestionnaireReminders: true, + receivesRecommendationUpdates: true, + receivesVitalsReminders: true, + receivesWeightAlerts: true, + }), + }); + } + + async function enrollAndFinish(): Promise { + const authUser = await env.auth.createUser({}); + await env.call( + enrollUser, + { invitationCode: "TESTCODE" }, + { uid: authUser.uid }, + ); + const userService = env.factory.user(); + const dbUser = await userService.getUser(authUser.uid); + expect(dbUser).toBeDefined(); + if (dbUser !== undefined) await userService.finishUserEnrollment(dbUser); + return authUser.uid; + } + + it("allows a permanent invitation to be reused by multiple users", async () => { + const invitationRef = env.collections.invitations.doc(); + await invitationRef.set(patientInvitation(true)); + + const seededAppointment = new FHIRAppointment({ + status: FHIRAppointmentStatus.booked, + created: new Date("2023-12-24"), + start: new Date("2023-12-31"), + end: new Date("2024-01-01"), + participant: [], + }); + await env.collections + .invitationAppointments(invitationRef.id) + .doc() + .set(seededAppointment); + + // First enrollee receives the seeded data and the invitation survives. + const firstUserId = await enrollAndFinish(); + + const invitationsAfterFirst = await env.collections.invitations.get(); + expect(invitationsAfterFirst.docs).toHaveLength(1); + const remainingAppointments = await env.collections + .invitationAppointments(invitationRef.id) + .get(); + expect(remainingAppointments.docs).toHaveLength(1); + const firstUserAppointments = await env.collections + .userAppointments(firstUserId) + .get(); + expect(firstUserAppointments.docs).toHaveLength(1); + + // Second enrollee can reuse the same code and also receives the seeded data. + const secondUserId = await enrollAndFinish(); + + const invitationsAfterSecond = await env.collections.invitations.get(); + expect(invitationsAfterSecond.docs).toHaveLength(1); + const secondUserAppointments = await env.collections + .userAppointments(secondUserId) + .get(); + expect(secondUserAppointments.docs).toHaveLength(1); + }); + + it("deletes a non-permanent invitation after enrollment", async () => { + const invitationRef = env.collections.invitations.doc(); + await invitationRef.set(patientInvitation(false)); + + await enrollAndFinish(); + + const invitations = await env.collections.invitations.get(); + expect(invitations.docs).toHaveLength(0); + }); }); diff --git a/functions/src/functions/exportData.test.ts b/functions/src/functions/exportData.test.ts index b9f9c6d0..c4d7a379 100644 --- a/functions/src/functions/exportData.test.ts +++ b/functions/src/functions/exportData.test.ts @@ -12,6 +12,9 @@ import { writeFileSync } from "fs"; import { CachingStrategy, DebugDataComponent, + DrugReference, + FHIRMedicationRequest, + fhirMedicationRequestConverter, StaticDataComponent, UserDebugDataComponent, UserObservationCollection, @@ -223,6 +226,124 @@ describeWithEmulators("function: exportData", (env) => { } }, 10_000); + it("reconstructs medication history with lifecycle dates", async () => { + const patient = await env.createUser({ + type: UserType.patient, + organization: "stanford", + }); + const admin = await env.createUser({ type: UserType.admin }); + + const history = env.factory.history(); + const encode = (request: FHIRMedicationRequest) => + fhirMedicationRequestConverter.value.encode(request); + + // Medication that was added and later removed: expect exactly one row with + // both `created` and `removed` set and no live document. + const removedId = "med-removed"; + const removedPath = `users/${patient}/medicationRequests/${removedId}`; + const removedMedication = FHIRMedicationRequest.create({ + medicationReference: DrugReference.carvedilol3_125, + frequencyPerDay: 1, + quantity: 1, + }); + await history.recordChange( + env.createChange(removedPath, undefined, encode(removedMedication)), + ); + await history.recordChange( + env.createChange(removedPath, encode(removedMedication), undefined), + ); + + // Medication whose dose changed and is still active: expect one row showing + // the final dose, `created` set and `removed` empty. + const changedId = "med-changed"; + const changedPath = `users/${patient}/medicationRequests/${changedId}`; + const initialDose = FHIRMedicationRequest.create({ + medicationReference: DrugReference.carvedilol3_125, + frequencyPerDay: 1, + quantity: 1, + }); + const finalDose = FHIRMedicationRequest.create({ + medicationReference: DrugReference.carvedilol3_125, + frequencyPerDay: 2, + quantity: 1, + }); + await history.recordChange( + env.createChange(changedPath, undefined, encode(initialDose)), + ); + await history.recordChange( + env.createChange(changedPath, encode(initialDose), encode(finalDose)), + ); + await env.collections + .userMedicationRequests(patient) + .doc(changedId) + .set(finalDose); + + // Medication that exists live but has no history (predates tracking): + // expect one row with empty lifecycle dates. + const liveOnlyId = "med-liveonly"; + const liveOnlyMedication = FHIRMedicationRequest.create({ + medicationReference: DrugReference.carvedilol25, + frequencyPerDay: 1, + quantity: 1, + }); + await env.collections + .userMedicationRequests(patient) + .doc(liveOnlyId) + .set(liveOnlyMedication); + + const result = await env.call( + exportData, + { userId: patient }, + { uid: admin, token: { type: UserType.admin } }, + ); + const zip = await yauzl.fromBuffer(Buffer.from(result.content, "base64")); + const entries = await zip.readEntries(); + const entry = entries.find( + (entry) => entry.filename === `${patient}/medicationRequests.csv`, + ); + expect(entry).toBeDefined(); + + const lines = (await entryBuffer(entry!)) + .toString("utf-8") + .split("\n") + .filter((line) => line.trim().length > 0); + + expect(lines[0].split(";")).toEqual([ + "id", + "medicationCode (RxNorm)", + "drugCode (RxNorm)", + "quantity", + "quantityUnit", + "frequencyPerDay", + "created", + "lastUpdated", + "removed", + ]); + + // One row per medication (add+remove collapses to a single row). + expect(lines).toHaveLength(4); + const rows = new Map( + lines.slice(1).map((line) => { + const columns = line.split(";"); + return [columns[0], columns]; + }), + ); + + const removedRow = rows.get(removedId)!; + expect(removedRow).toBeDefined(); + expect(removedRow[6]).not.toBe(""); // created + expect(removedRow[8]).not.toBe(""); // removed + + const changedRow = rows.get(changedId)!; + expect(changedRow[5]).toBe("2"); // final frequencyPerDay + expect(changedRow[6]).not.toBe(""); // created + expect(changedRow[8]).toBe(""); // still active, not removed + + const liveOnlyRow = rows.get(liveOnlyId)!; + expect(liveOnlyRow[6]).toBe(""); // no history -> empty created + expect(liveOnlyRow[8]).toBe(""); // not removed + }, 10_000); + async function expectZipToBeEquivalent( file0: ZipFile, file1: ZipFile, diff --git a/functions/src/services/export/defaultExportService.ts b/functions/src/services/export/defaultExportService.ts index 950b94d9..6f9b7d17 100644 --- a/functions/src/services/export/defaultExportService.ts +++ b/functions/src/services/export/defaultExportService.ts @@ -7,6 +7,7 @@ // import { + fhirMedicationRequestConverter, type FHIRQuestionnaireItem, LoincCode, UserObservationCollection, @@ -14,7 +15,10 @@ import { import archiver, { type Archiver } from "archiver"; import { https } from "firebase-functions/v2"; import { type ExportService } from "./exportService.js"; -import { type DatabaseService } from "../database/databaseService.js"; +import { + type DatabaseService, + type Document, +} from "../database/databaseService.js"; import { QuestionnaireId, QuestionnaireLinkId, @@ -31,6 +35,19 @@ interface UserCsvExport { readonly csv: CsvData; } +/** + * A single document reconstructed across its lifecycle from the global `/history` + * collection (merged with the live collection). One record per document, so a + * medication that was added and later removed yields exactly one row. + */ +interface LifecycleRecord { + readonly id: string; + readonly content: Content; + readonly created?: Date; + readonly lastUpdated?: Date; + readonly removed?: Date; +} + export class DefaultExportService implements ExportService { private readonly databaseService: DatabaseService; private readonly userService: UserService; @@ -240,9 +257,15 @@ export class DefaultExportService implements ExportService { userId: string, archiver: Archiver, ): Promise { - const medications = await this.databaseService.getQuery((collections) => + const liveDocs = await this.databaseService.getQuery((collections) => collections.userMedicationRequests(userId), ); + const records = await this.lifecycleRecords( + userId, + "medicationRequests", + liveDocs, + (data) => fhirMedicationRequestConverter.value.schema.parse(data), + ); const csv = this.createCsvData( [ @@ -252,24 +275,31 @@ export class DefaultExportService implements ExportService { "quantity", "quantityUnit", "frequencyPerDay", + "created", + "lastUpdated", + "removed", ], - medications, - (value) => { + records, + (record) => { + const medication = record.content; const referenceParts = ( - value.content.medicationReference?.reference ?? "" + medication.medicationReference?.reference ?? "" ).split("/"); - const quantity = value.content.dosageInstruction + const quantity = medication.dosageInstruction ?.at(0) ?.doseAndRate?.at(0)?.doseQuantity; const frequency = - value.content.dosageInstruction?.at(0)?.timing?.repeat?.frequency; + medication.dosageInstruction?.at(0)?.timing?.repeat?.frequency; return [ - value.id, + record.id, referenceParts[1], referenceParts[3], quantity?.value?.toString() ?? "", quantity?.unit ?? "", frequency?.toString() ?? "", + record.created?.toISOString() ?? "", + record.lastUpdated?.toISOString() ?? "", + record.removed?.toISOString() ?? "", ]; }, ); @@ -280,6 +310,76 @@ export class DefaultExportService implements ExportService { return { filename: "medicationRequests.csv", csv }; } + /** + * Reconstructs the lifecycle of every document in a user's sub-collection by + * reading the global `/history` collection (range-queried by document path) + * and merging it with the live collection. Returns one record per document: + * - `created`: earliest history entry date (empty if the doc predates history + * tracking and only exists live). + * - `lastUpdated`: latest history entry with non-null data. + * - `removed`: date the document was deleted (it no longer exists live). + * - `content`: the live content if it still exists, otherwise the most recent + * non-null history snapshot parsed back into the model. + */ + private async lifecycleRecords( + userId: string, + subcollection: string, + liveDocs: Array>, + parse: (data: unknown) => Content, + ): Promise>> { + const prefix = `users/${userId}/${subcollection}/`; + const historyEntries = await this.databaseService.getQuery((collections) => + collections.history + .where("path", ">=", prefix) + .where("path", "<", prefix + "\uf8ff"), + ); + + const groups = new Map>(); + for (const entry of historyEntries) { + const group = groups.get(entry.content.path) ?? []; + group.push({ date: entry.content.date, data: entry.content.data }); + groups.set(entry.content.path, group); + } + + const liveById = new Map(liveDocs.map((doc) => [doc.id, doc])); + const records: Array> = []; + const seenIds = new Set(); + + for (const [path, entries] of groups) { + const id = path.substring(path.lastIndexOf("/") + 1); + seenIds.add(id); + entries.sort((a, b) => a.date.getTime() - b.date.getTime()); + + const nonNullEntries = entries.filter((entry) => entry.data != null); + const created = entries.at(0)?.date; + const lastUpdated = nonNullEntries.at(-1)?.date; + const liveDoc = liveById.get(id); + + let content: Content | undefined; + let removed: Date | undefined; + if (liveDoc !== undefined) { + content = liveDoc.content; + } else { + const latestData = nonNullEntries.at(-1)?.data; + content = latestData != null ? parse(latestData) : undefined; + removed = entries.at(-1)?.date; + } + + // No recoverable data (e.g. only a deletion entry exists) - skip. + if (content === undefined) continue; + + records.push({ id, content, created, lastUpdated, removed }); + } + + // Include live documents that predate history tracking (no history entries). + for (const doc of liveDocs) { + if (seenIds.has(doc.id)) continue; + records.push({ id: doc.id, content: doc.content }); + } + + return records; + } + private async addUserObservations( userId: string, collection: UserObservationCollection, diff --git a/functions/src/services/user/databaseUserService.ts b/functions/src/services/user/databaseUserService.ts index 6cdfb0a9..df856f37 100644 --- a/functions/src/services/user/databaseUserService.ts +++ b/functions/src/services/user/databaseUserService.ts @@ -225,6 +225,10 @@ export class DatabaseUserService implements UserService { `DatabaseUserService.finishUserEnrollment(${user.id}): Will copy invitation collections: [${invitationCollections.map((collection) => `'${collection.id}'`).join(", ")}].`, ); + // Permanent invitations can be reused: keep the invitation document and its + // seeded sub-collection data so the next invitee receives the same data. + const isPermanent = invitation.content.permanent; + await Promise.all( invitationCollections.map(async (invitationCollection) => this.databaseService.runTransaction( @@ -237,7 +241,9 @@ export class DatabaseUserService implements UserService { userRef.collection(collectionId).doc(item.id), item.data(), ); - transaction.delete(item.ref); + if (!isPermanent) { + transaction.delete(item.ref); + } } logger.info( @@ -248,7 +254,9 @@ export class DatabaseUserService implements UserService { ), ); - await this.deleteInvitation(invitation); + if (!isPermanent) { + await this.deleteInvitation(invitation); + } } async deleteInvitation(invitation: Document): Promise { diff --git a/functions/src/tests/resources/patientExport.zip b/functions/src/tests/resources/patientExport.zip index 66403dea..959d0139 100644 --- a/functions/src/tests/resources/patientExport.zip +++ b/functions/src/tests/resources/patientExport.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:50e3dc19b17dc558ee86bb9e97d24dfd60397c4ae2883ad4f8f99f624c78b292 -size 26925 +oid sha256:a3592a6bd5e0a7961a92695296c931e1ed96b0daacc78d3f294d9e4eefe78373 +size 27096