diff --git a/Dockerfile b/Dockerfile index 0c0ebed02..318325eff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,6 +66,12 @@ RUN cd ${MAGE_SERVER}/plugins/nga-msi \ && cd ${MAGE_PACKAGES} \ && npm pack ${MAGE_SERVER}/plugins/nga-msi +RUN cd ${MAGE_SERVER}/plugins/clamav/service \ + && npm link ../../../service \ + && npm run build \ + && cd ${MAGE_PACKAGES} \ + && npm pack ${MAGE_SERVER}/plugins/clamav/service + WORKDIR ${MAGE_INSTANCE} RUN cd ${MAGE_INSTANCE} \ && npm install --omit dev --force \ @@ -75,6 +81,7 @@ RUN cd ${MAGE_INSTANCE} \ ${MAGE_PACKAGES}/ngageoint-mage.arcgis.*.tgz \ ${MAGE_PACKAGES}/ngageoint-mage.image.*.tgz \ ${MAGE_PACKAGES}/ngageoint-mage.nga-msi-*.tgz \ + ${MAGE_PACKAGES}/ngageoint-mage.clamav.*.tgz \ && ln -s ./node_modules/.bin/mage.service FROM ${DIST_IMAGE} diff --git a/docker-compose.yml b/docker-compose.yml index 86c3fe76b..5d11be7a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,16 @@ services: + # Add ClamAV to build local container for testing + mage-clamav: + image: clamav/clamav:stable + platform: linux/amd64 + volumes: + - ./clamav-test/data:/var/lib/clamav + ports: + - "3310:3310" + networks: + - mage.net + mage-db: image: mongodb/mongodb-community-server:8.0-ubi9-slim # Uncomment the following ports block to allow the mongo client on your @@ -34,7 +45,11 @@ services: environment: MAGE_MONGO_URL: mongodb://mage-db:27017/magedb MAGE_TOKEN_EXPIRATION: "28800" - MAGE_PLUGINS: '{"servicePlugins":["@ngageoint/mage.sftp.service", "@ngageoint/mage.arcgis.service"],"webUIPlugins":["@ngageoint/mage.sftp.web", "@ngageoint/mage.arcgis.web-app"]}' + MAGE_PLUGINS: '{"servicePlugins":["@ngageoint/mage.sftp.service","@ngageoint/mage.arcgis.service","@ngageoint/mage.clamav.service"],"webUIPlugins":["@ngageoint/mage.sftp.web","@ngageoint/mage.arcgis.web-app"]}' + # Only used if @ngageoint/mage.clamav.service is enabled above - host/port + # for the mage-clamav container this compose file already defines. + MAGE_CLAMAV_HOST: mage-clamav + MAGE_CLAMAV_PORT: "3310" MAGE_SFTP_KEY_FILE: /var/lib/mage/sftp_key # Uncomment this section to use Docker volumes instead of bind mounts for data directories: diff --git a/plugins/clamav/service/package.json b/plugins/clamav/service/package.json new file mode 100644 index 000000000..8bb659183 --- /dev/null +++ b/plugins/clamav/service/package.json @@ -0,0 +1,37 @@ +{ + "name": "@ngageoint/mage.clamav.service", + "version": "1.0.0-beta.1", + "description": "The ClamAV service package is a Mage server plugin that scans attachment content for viruses before it is made available.", + "main": "lib/index.js", + "scripts": { + "build": "tsc", + "test": "npm-run-all build test:run", + "test:run": "jasmine" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ngageoint/mage-server.git" + }, + "keywords": [ + "mage", + "plugin", + "clamav", + "attachment-processing" + ], + "author": "National Geospatial-Intelligence Agency", + "license": "MIT", + "bugs": { + "url": "https://github.com/ngageoint/mage-server/issues" + }, + "homepage": "https://github.com/ngageoint/mage-server#readme", + "devDependencies": { + "@types/jasmine": "^3.10.6", + "@types/node": "^26.1.0", + "jasmine": "^6.3.0", + "npm-run-all": "^4.1.5", + "typescript": "^5.9.3" + }, + "peerDependencies": { + "@ngageoint/mage.service": "^6.5.8 || ^6.6.0-beta" + } +} diff --git a/plugins/clamav/service/src/clamHook.ts b/plugins/clamav/service/src/clamHook.ts new file mode 100644 index 000000000..36b410479 --- /dev/null +++ b/plugins/clamav/service/src/clamHook.ts @@ -0,0 +1,24 @@ +import fs from 'fs' +import { scan } from './scan' +import { AttachmentHook } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.attachments' + +// ClamAV hook start +export const clamavHook: AttachmentHook = async (attachment, stagedFilePath) => { + const host = process.env.MAGE_CLAMAV_HOST + const port = process.env.MAGE_CLAMAV_PORT ? Number(process.env.MAGE_CLAMAV_PORT) : undefined + + // Outcome from stream + const outcome = await scan(fs.createReadStream(stagedFilePath), { host, port }) + + // Message return for a rejected statement + if (outcome.outcome === 'reject' ) { + return { outcome: 'reject', reason: `This file/attachment was flagged as potentially malicious and could not be uploaded - (${outcome.reason})` } + } + + // Message return for errors (not including malicious attachments) + if (outcome.outcome === 'error') { + return { outcome: 'error', error: new Error(`Virus scan could not be completed - (${outcome.error.message})`, { cause: outcome.error }) } + } + + return outcome +} \ No newline at end of file diff --git a/plugins/clamav/service/src/index.ts b/plugins/clamav/service/src/index.ts new file mode 100644 index 000000000..c01c25098 --- /dev/null +++ b/plugins/clamav/service/src/index.ts @@ -0,0 +1,11 @@ +import { InitPluginHook } from '@ngageoint/mage.service/lib/plugins.api' +import { clamavHook } from './clamHook' + +const clamavPluginHooks: InitPluginHook = { + init: async () => { + return { attachmentHooks: [clamavHook] } + } +} + +// One object for export +export = clamavPluginHooks \ No newline at end of file diff --git a/plugins/clamav/service/src/scan.ts b/plugins/clamav/service/src/scan.ts new file mode 100644 index 000000000..7f0a59a13 --- /dev/null +++ b/plugins/clamav/service/src/scan.ts @@ -0,0 +1,110 @@ +import net from 'net' +import { AttachmentHookOutcome } from '@ngageoint/mage.service/lib/plugins.api/plugins.api.attachments' + +const DEFAULT_TIMEOUT_MS = 15_000 + +export type ScanOptions = { + host?: string + port?: number + timeoutMs?: number + createConnection?: (options: { host: string, port: number }) => net.Socket +} + +// Scan function +export function scan(inputStream: NodeJS.ReadableStream & { destroy: (error?: Error) => void }, options: ScanOptions = {}): Promise { + const { + host = 'localhost', + port = 3310, + timeoutMs = DEFAULT_TIMEOUT_MS, + createConnection = net.createConnection, + } = options + + return new Promise((resolve) => { + const socket = createConnection({ host, port }) + + // Pause immediately, before wiring up any listeners, so the command is + // always sent before any input bytes reach clamd, regardless of which + // async operation (socket connect vs. stream read) happens to finish first. + inputStream.pause() + + const responseChunks: Buffer[] = [] + let settled = false + + const timeout = setTimeout(() => finish(errorOutcome(new Error('scan timed out'))), timeoutMs) + + function finish(outcome: AttachmentHookOutcome) { + if (settled) return + settled = true + clearTimeout(timeout) + inputStream.destroy() + socket.destroy() + resolve(outcome) + } + + socket.on('error', (err: Error) => finish(errorOutcome(new Error(`socket error: ${err.message}`)))) + + socket.on('connect', () => { + socket.write('zINSTREAM\0') + inputStream.resume() + }) + + inputStream.on('error', (err: Error) => finish(errorOutcome(new Error(`input stream error: ${err.message}`)))) + + inputStream.on('data', (chunk: Buffer) => { + // Each chunk is preceded by its own length as a 4-byte big-endian + // unsigned integer, per the INSTREAM protocol. + const lenPrefix = Buffer.alloc(4) + lenPrefix.writeUInt32BE(chunk.length, 0) + const stillHasRoom = socket.write(Buffer.concat([lenPrefix, chunk])) + + // Honor backpressure: pause reading more input until the socket + // signals it has caught up. + if (!stillHasRoom) { + inputStream.pause() + socket.once('drain', () => inputStream.resume()) + } + }) + + inputStream.on('end', () => { + // A zero-length chunk tells clamd no more data is coming. + socket.write(Buffer.alloc(4)) + }) + + socket.on('data', (chunk: Buffer) => responseChunks.push(chunk)) + + socket.on('end', () => { + // clamd's reply to a z-prefixed command (like zINSTREAM) is + // NUL-terminated, not newline-terminated - strip NUL bytes before + // trimming whitespace. + const response = Buffer.concat(responseChunks).toString().replace(/\0/g, '').trim() + finish(parseResponse(response)) + }) + }) +} + +// clamd's INSTREAM reply is always "stream: ": +// - "stream: OK" - clean +// - "stream: FOUND" - infected +// - anything else is treated as an error rather than guessed at +function parseResponse(response: string): AttachmentHookOutcome { + const match = /^stream:\s*(.*)$/.exec(response) + if (!match) { + return errorOutcome(new Error(`unrecognized clamd response: "${response}"`)) + } + + const result = match[1].trim() + if (result === 'OK') { + return { outcome: 'pass' } + } + + const foundMatch = /^(.+) FOUND$/.exec(result) + if (foundMatch) { + return { outcome: 'reject', reason: foundMatch[1] } + } + + return errorOutcome(new Error(`unrecognized clamd result: "${result}"`)) +} + +function errorOutcome(error: Error): AttachmentHookOutcome { + return { outcome: 'error', error } +} \ No newline at end of file diff --git a/plugins/clamav/service/tsconfig.json b/plugins/clamav/service/tsconfig.json new file mode 100644 index 000000000..8d70d38f8 --- /dev/null +++ b/plugins/clamav/service/tsconfig.json @@ -0,0 +1,17 @@ +{ + "include": ["src/**/*.ts", "src/**/*.spec.ts", "src/**/*.test.ts"], + "exclude": ["node_modules", "lib", "dist"], + "compilerOptions": { + "strict": true, + "outDir": "lib", + "target": "ES2022", + "lib": ["ES2022"], + "sourceMap": true, + "module": "commonjs", + "moduleResolution": "node", + "esModuleInterop": true, + "baseUrl": ".", + "types": ["jasmine", "node"], + "skipLibCheck": true + } +} diff --git a/plugins/image/service/src/processor.spec.ts b/plugins/image/service/src/processor.spec.ts index b38ecbcc1..050367785 100644 --- a/plugins/image/service/src/processor.spec.ts +++ b/plugins/image/service/src/processor.spec.ts @@ -945,6 +945,15 @@ class BufferAttachmentStore implements AttachmentStore { this.pendingContent.set(id, Buffer.alloc(0)) return pending } + async stagedContentPath(stagedContentId: StagedAttachmentContentId): Promise { + if (!this.pendingContent.has(stagedContentId as string)) { + return new AttachmentStoreError(AttachmentStoreErrorCode.ContentNotFound, `pending content not found: ${stagedContentId}`) + } + return stagedContentId as string + } + async deleteStagedContent(stagedContentId: StagedAttachmentContentId): Promise { + this.pendingContent.delete(stagedContentId as string) + } async saveContent(content: NodeJS.ReadableStream | StagedAttachmentContent, attachmentId: string, observation: Observation): Promise { if (typeof content !== 'string') { return new AttachmentStoreError(AttachmentStoreErrorCode.ContentNotFound, 'this store supports saving only staged content') diff --git a/service/src/adapters/observations/adapters.observations.attachment_store.file_system.ts b/service/src/adapters/observations/adapters.observations.attachment_store.file_system.ts index 740343303..88ff1feb4 100644 --- a/service/src/adapters/observations/adapters.observations.attachment_store.file_system.ts +++ b/service/src/adapters/observations/adapters.observations.attachment_store.file_system.ts @@ -3,7 +3,7 @@ import path from 'path' import stream from 'stream' import util from 'util' import uniqid from 'uniqid' -import { Attachment, AttachmentStore, AttachmentStoreError, AttachmentStoreErrorCode, copyThumbnailAttrs, Observation, patchAttachment, StagedAttachmentContent, StagedAttachmentContentRef, Thumbnail, AttachmentContentPatchAttrs, ThumbnailContentPatchAttrs, AttachmentPatchAttrs } from '../../entities/observations/entities.observations' +import { Attachment, AttachmentStore, AttachmentStoreError, AttachmentStoreErrorCode, copyThumbnailAttrs, Observation, patchAttachment, StagedAttachmentContent, StagedAttachmentContentRef, Thumbnail, AttachmentContentPatchAttrs, ThumbnailContentPatchAttrs, AttachmentPatchAttrs, StagedAttachmentContentId } from '../../entities/observations/entities.observations' import mime from 'mime-types' export class FileSystemAttachmentStore implements AttachmentStore { @@ -21,6 +21,28 @@ export class FileSystemAttachmentStore implements AttachmentStore { return new StagedAttachmentContent(id, tempLocation) } + // Define the staged content path + async stagedContentPath(stagedContentId: StagedAttachmentContentId): Promise { + return path.join(this.pendingDirPath, stagedContentId as string) + } + + // Define delete staged content + async deleteStagedContent(stagedContentId: StagedAttachmentContentId): Promise { + const tempPath = path.join(this.pendingDirPath, stagedContentId as string) + try{ + // Cleanly removing path + await util.promisify(fs.rm)(tempPath) + + } catch (err) { + // Message for crashes or any failures + const message = `error deleting staged content ${stagedContentId}` + + // Add console output + console.error(message, err) + return new AttachmentStoreError(AttachmentStoreErrorCode.StorageError, `${message}: ${err instanceof Error ? err.message : String(err)}`) + } + } + async saveContent(content: StagedAttachmentContentRef | NodeJS.ReadableStream, attachmentId: string, observation: Observation): Promise { const attachment = observation.attachmentFor(attachmentId) if (!attachment) { diff --git a/service/src/adapters/observations/adapters.observations.db.mongoose.ts b/service/src/adapters/observations/adapters.observations.db.mongoose.ts index 5e6c2e732..ba9dd4ce1 100644 --- a/service/src/adapters/observations/adapters.observations.db.mongoose.ts +++ b/service/src/adapters/observations/adapters.observations.db.mongoose.ts @@ -146,6 +146,13 @@ export function docToEntity(doc: legacy.ObservationDocument, eventId: MageEventI return createDocumentMapping(eventId)(doc) } +// Define what a pending attachment looks like +export type PendingAttachmentReference = { + eventId: MageEventId + observationId: ObservationId + attachmentId: AttachmentId +} + function createDocumentMapping(eventId: MageEventId): DocumentMapping { return doc => { const attrs: ObservationAttrs = { @@ -205,6 +212,11 @@ function attachmentAttrsForDoc(doc: legacy.AttachmentDocument): Attachment { oriented: doc.oriented, contentLocator: doc.relativePath, thumbnails: doc.thumbnails.map(thumbnailAttrsForDoc), + processingStatus: doc.processingStatus, + processingMessage: doc.processingMessage, + processingHook: doc.processingHook, + stagedContentId: doc.stagedContentId, + processingRetryCount: doc.processingRetryCount, } } @@ -266,3 +278,42 @@ function thumbnailDocSeedForEntity(attrs: Thumbnail): legacy.ThumbnailDocAttrs { height: attrs.height, } } + +// Define the function to check for pending attachments +export async function findPendingAttachments(limit: number): Promise { + + // Set the array of documents + const eventDocs = await mongoose.connection.collection<{ _id: number, collectionName: string }>('events').find({}).toArray() + + // Loop through each document + const references: PendingAttachmentReference[] = [] + for (const eventDoc of eventDocs) { + if (references.length >= limit) { + break + } + const model = legacy.observationModel({ id: eventDoc._id, collectionName: eventDoc.collectionName }) + + // mongo aggregation pipeline + const remaining = limit - references.length + const pipeline = [ + { $match: { 'attachments.processingStatus': 'pending' } }, + { $unwind: '$attachments' }, + { $match: { 'attachments.processingStatus': 'pending' } }, + { $project: { _id: false, observationId: '$_id', attachmentId: '$attachments._id' } }, + { $limit: remaining }, + ] + + // Run the pipeline against the model + const matches = await model.aggregate(pipeline) + + // loop over the matches + for (const match of matches) { + references.push({ + eventId: eventDoc._id, + observationId: match.observationId.toString(), + attachmentId: match.attachmentId.toString() + }) + } + } + return references +} diff --git a/service/src/app.impl/observations/app.impl.observations.ts b/service/src/app.impl/observations/app.impl.observations.ts index a335532f7..8f4c72117 100644 --- a/service/src/app.impl/observations/app.impl.observations.ts +++ b/service/src/app.impl/observations/app.impl.observations.ts @@ -1,11 +1,16 @@ import EventEmitter from 'events' +import stream from 'stream' +import util from 'util' import { entityNotFound, infrastructureError, invalidInput, InvalidInputError, MageError } from '../../app.api/app.api.errors' import { AppResponse } from '../../app.api/app.api.global' import * as api from '../../app.api/observations/app.api.observations' import { MageEvent } from '../../entities/events/entities.events' import { FormFieldType } from '../../entities/events/entities.events.forms' -import { addAttachment, AttachmentCreateAttrs, AttachmentNotFoundError, AttachmentsRemovedDomainEvent, AttachmentStore, AttachmentStoreError, AttachmentStoreErrorCode, FormEntry, FormEntryId, FormFieldEntry, Observation, ObservationAttrs, ObservationDomainEventType, ObservationEmitted, ObservationRepositoryErrorCode, removeAttachment, thumbnailIndexForTargetDimension, validationResultMessage } from '../../entities/observations/entities.observations' +import { addAttachment, AttachmentContentPatchAttrs, AttachmentCreateAttrs, AttachmentNotFoundError, AttachmentPatchAttrs, AttachmentsRemovedDomainEvent, AttachmentStore, AttachmentStoreError, AttachmentStoreErrorCode, FormEntry, FormEntryId, FormFieldEntry, Observation, ObservationAttrs, ObservationDomainEventType, ObservationEmitted, ObservationRepositoryErrorCode, removeAttachment, StagedAttachmentContentRef, thumbnailIndexForTargetDimension, validationResultMessage } from '../../entities/observations/entities.observations' import { UserId, UserRepository } from '../../entities/users/entities.users' +import { AttachmentHook } from '../../plugins.api/plugins.api.attachments' + +const pipeline = util.promisify(stream.pipeline) export function AllocateObservationId(permissionService: api.ObservationPermissionService): api.AllocateObservationId { return async function allocateObservationId(req: api.AllocateObservationIdRequest): ReturnType { @@ -52,7 +57,7 @@ export function SaveObservation(permissionService: api.ObservationPermissionServ } } -export function StoreAttachmentContent(permissionService: api.ObservationPermissionService, attachmentStore: AttachmentStore): api.StoreAttachmentContent { +export function StoreAttachmentContent(permissionService: api.ObservationPermissionService, attachmentStore: AttachmentStore, attachmentHooks: AttachmentHook[]): api.StoreAttachmentContent { return async function storeAttachmentContent(req: api.StoreAttachmentContentRequest): ReturnType { const obsRepo = req.context.observationRepository const obsBefore = await obsRepo.findById(req.observationId) @@ -72,7 +77,43 @@ export function StoreAttachmentContent(permissionService: api.ObservationPermiss if (denied) { return AppResponse.error(denied) } - const attachmentPatch = await attachmentStore.saveContent(req.content.bytes, attachmentBefore.id, obsBefore) + let attachmentPatch: AttachmentContentPatchAttrs | AttachmentPatchAttrs | AttachmentStoreError | null + if (attachmentHooks.length === 0) { + /* + No attachment-processing hooks are registered (e.g. no ClamAV or other + plugin enabled) - keep the original, direct-to-final-storage behavior + unchanged, so installations that never asked for attachment processing + see no staging latency at all. + */ + attachmentPatch = await attachmentStore.saveContent(req.content.bytes, attachmentBefore.id, obsBefore) + } + else if (req.content.bytes instanceof StagedAttachmentContentRef) { + /* + The incoming content is already a reference to previously-staged + content (not a fresh stream) - nothing to write here, just record its + existing id so the background job can find it later. + */ + attachmentPatch = { processingStatus: 'pending', stagedContentId: req.content.bytes.id as string } + } + else { + /* + Hooks are registered and this is a fresh stream - stage the upload + instead of finalizing it immediately. A later, separate attachment- + processing job runs the registered hooks against the staged content + and only then finalizes (saveContent with a StagedAttachmentContentRef) + or rejects it. + */ + const staged = await attachmentStore.stagePendingContent() + try { + await pipeline(req.content.bytes, staged.tempLocation) + attachmentPatch = { processingStatus: 'pending', stagedContentId: staged.id as string } + } + catch (err) { + const message = `error staging attachment content for attachment ${attachmentBefore.id} on observation ${obsBefore.id}` + console.error(message, err) + attachmentPatch = new AttachmentStoreError(AttachmentStoreErrorCode.StorageError, `${message}: ${err instanceof Error ? err.message : String(err)}`) + } + } if (attachmentPatch instanceof AttachmentStoreError) { if (attachmentPatch.errorCode === AttachmentStoreErrorCode.StorageError) { return AppResponse.error(infrastructureError(attachmentPatch)) diff --git a/service/src/app.ts b/service/src/app.ts index 1d50deaa5..4d93d1356 100644 --- a/service/src/app.ts +++ b/service/src/app.ts @@ -7,7 +7,7 @@ import { } from './main.impl/main.impl.plugins'; import httpLib from 'http'; import fs from 'fs-extra'; -import mongoose from 'mongoose'; +import mongoose, { plugin } from 'mongoose'; import express from 'express'; import util from 'util'; import { @@ -150,6 +150,10 @@ import { import { RoleBasedMapPermissionService } from './permissions/permissions.settings'; import { SettingRepository } from './entities/settings/entities.settings'; +// Attachment imports +import { AttachmentHook } from './plugins.api/plugins.api.attachments'; +import { startAttachmentProcessing } from './main.impl/main.impl.attachment_processing'; + export interface MageService { webController: express.Application; server: httpLib.Server; @@ -226,7 +230,17 @@ export const boot = async function(config: BootConfig): Promise { const dbLayer = await initDatabase(); const repos = await initRepositories(dbLayer, config); - const appLayer = await initAppLayer(repos); + /* + Declared here, before plugins load, and passed by reference into + initAppLayer/storeAttachmentContent below. storeAttachmentContent is only + actually invoked per HTTP request, long after boot() finishes, so it's + safe for this array to still be empty right now - it gets filled in below, + after the plugin-loading loop, by mutating this same array in place + (not reassigning it), so the reference already handed to + storeAttachmentContent's closure sees the final contents. + */ + const attachmentHooks: AttachmentHook[] = []; + const appLayer = await initAppLayer(repos, attachmentHooks); const { webController, addPluginRoutes } = await initWebLayer( repos, appLayer, @@ -241,6 +255,15 @@ export const boot = async function(config: BootConfig): Promise { routesForPluginId[pluginId] = initPluginRoutes; }; + // Hooks by plugin + const attachmentHooksByPluginId: { [pluginId: string]: AttachmentHook[] } = {}; + const collectAttachmentHooks = ( + pluginId: string, + attachmentHooks: AttachmentHook[] + ): void => { + attachmentHooksByPluginId[pluginId] = attachmentHooks; + }; + const globalScopeServices = new Map, any>([ [FeedServiceTypeRepositoryToken, repos.feeds.serviceTypeRepo], [FeedServiceRepositoryToken, repos.feeds.serviceRepo], @@ -288,13 +311,25 @@ export const boot = async function(config: BootConfig): Promise { pluginId, initPlugin, injectService, - collectPluginRoutesToSort + collectPluginRoutesToSort, + collectAttachmentHooks ); } catch (err) { console.error(`error loading plugin ${pluginId}`, err); } } + // Flatten hooks in array. Mutates the array declared earlier (not a + // reassignment) so the reference already passed into storeAttachmentContent + // reflects these contents once real requests start coming in. + attachmentHooks.push(...Object.values(attachmentHooksByPluginId).flat()) + + // Start the background job that finds attachments staged by + // storeAttachmentContent and runs them through the now-final attachmentHooks + // list. Core-owned (not tied to any one plugin's init()), since it must run + // hooks contributed by any enabled plugin. + startAttachmentProcessing(repos.observations.obsRepoFactory, repos.observations.attachmentStore, attachmentHooks, console); + const pluginRoutePathsDescending = Object.keys(routesForPluginId) .sort() .reverse(); @@ -568,9 +603,9 @@ async function initRepositories( }; } -async function initAppLayer(repos: Repositories): Promise { +async function initAppLayer(repos: Repositories, attachmentHooks: AttachmentHook[]): Promise { const events = await initEventsAppLayer(repos); - const observations = await initObservationsAppLayer(repos); + const observations = await initObservationsAppLayer(repos, attachmentHooks); const icons = await initIconsAppLayer(repos); const feeds = await initFeedsAppLayer(repos); const users = await initUsersAppLayer(repos); @@ -635,7 +670,8 @@ async function initEventsAppLayer( } async function initObservationsAppLayer( - repos: Repositories + repos: Repositories, + attachmentHooks: AttachmentHook[] ): Promise { const eventPermissions = await import('./permissions/permissions.events'); const obsPermissions = await import('./permissions/permissions.observations'); @@ -659,7 +695,8 @@ async function initObservationsAppLayer( ), storeAttachmentContent: observationsImpl.StoreAttachmentContent( obsPermissionsService, - repos.observations.attachmentStore + repos.observations.attachmentStore, + attachmentHooks ), readAttachmentContent: observationsImpl.ReadAttachmentContent( obsPermissionsService, diff --git a/service/src/entities/observations/entities.observations.ts b/service/src/entities/observations/entities.observations.ts index c90603630..9486919f9 100644 --- a/service/src/entities/observations/entities.observations.ts +++ b/service/src/entities/observations/entities.observations.ts @@ -91,6 +91,7 @@ export interface FormEntry { export type FormFieldEntryItem = Exclude | Geometry | Date export type FormFieldEntry = FormFieldEntryItem | FormFieldEntryItem[] | null +export type AttachmentProcessingStatus = 'pending' | 'clean' | 'rejected' | 'error' export type AttachmentId = string /** * TODO: Currently the web app uses the `name` and `contentType` keys in the @@ -150,6 +151,20 @@ export interface Attachment { */ oriented: boolean thumbnails: Thumbnail[] + processingStatus?: AttachmentProcessingStatus + processingMessage?: string + processingHook?: string + /** + * The ID of this attachment's staged content, if any, returned from + * {@link AttachmentStore.stagePendingContent}. Persisted so a later, + * separate process (e.g. a background attachment-processing job) can find + * and finalize or discard the staged file, since the original upload + * request's local reference to it does not survive past that request. + */ + stagedContentId?: string + + // + processingRetryCount?: number } export interface Thumbnail { @@ -202,7 +217,12 @@ export function copyAttachmentAttrs(from: Attachment): Attachment { height: from.height, oriented: from.oriented, thumbnails: from.thumbnails.map(copyThumbnailAttrs), - contentLocator: from.contentLocator + contentLocator: from.contentLocator, + processingStatus: from.processingStatus, + processingMessage: from.processingMessage, + processingHook: from.processingHook, + stagedContentId: from.stagedContentId, + processingRetryCount: from.processingRetryCount } } @@ -726,6 +746,11 @@ export function patchAttachment(observation: Observation, attachmentId: Attachme patched.size = patchHasProperty('size') ? patch.size : patched.size patched.contentLocator = patchHasProperty('contentLocator') ? patch.contentLocator : patched.contentLocator patched.thumbnails = patchHasProperty('thumbnails') ? patch.thumbnails?.map(copyThumbnailAttrs) as Thumbnail[] : patched.thumbnails + patched.processingStatus = patchHasProperty('processingStatus') ? patch.processingStatus : patched.processingStatus + patched.processingMessage = patchHasProperty('processingMessage') ? patch.processingMessage : patched.processingMessage + patched.processingHook = patchHasProperty('processingHook') ? patch.processingHook : patched.processingHook + patched.stagedContentId = patchHasProperty('stagedContentId') ? patch.stagedContentId : patched.stagedContentId + patched.processingRetryCount = patchHasProperty('processingRetryCount') ? patch.processingRetryCount : patched.processingRetryCount patched.lastModified = new Date() const patchedObservation = copyObservationAttrs(observation) const before = patchedObservation.attachments.slice(0, targetPos) @@ -966,6 +991,12 @@ export interface AttachmentStore { * the attachment. */ deleteContent(attachment: Attachment, observation: Observation): Promise + + // Return real file path for attachment + stagedContentPath(stagedContentId: StagedAttachmentContentId): Promise + + // Delete staged content + deleteStagedContent(stagedContentId: StagedAttachmentContentId): Promise } export class AttachmentStoreError extends Error { diff --git a/service/src/main.impl/main.impl.attachment_processing.ts b/service/src/main.impl/main.impl.attachment_processing.ts new file mode 100644 index 000000000..7884afec4 --- /dev/null +++ b/service/src/main.impl/main.impl.attachment_processing.ts @@ -0,0 +1,144 @@ +import { AttachmentHook, runPipeline } from '../plugins.api/plugins.api.attachments' +import { AttachmentPatchAttrs, AttachmentStore, AttachmentStoreError, ObservationRepositoryForEvent, StagedAttachmentContentRef } from '../entities/observations/entities.observations' +import { findPendingAttachments, PendingAttachmentReference } from '../adapters/observations/adapters.observations.db.mongoose' + +export type AttachmentProcessingConfig = { + intervalSeconds: number + batchSize: number + retryLimit: number +} + +export const defaultAttachmentProcessingConfig: AttachmentProcessingConfig = { + intervalSeconds: 15, + batchSize: 20, + retryLimit: 3 +} + +/** + * Runs the registered attachment-processing hooks against one pending + * attachment, then applies whatever patch the outcome calls for. Never + * throws - any unexpected error is logged and the attachment is left as-is + * to be retried on the next cycle. + */ +async function processPendingAttachment( + reference: PendingAttachmentReference, + obsRepoForEvent: ObservationRepositoryForEvent, + attachmentStore: AttachmentStore, + attachmentHooks: AttachmentHook[], + retryLimit: number, + console: Console +): Promise { + try { + const obsRepo = await obsRepoForEvent(reference.eventId) + const observation = await obsRepo.findById(reference.observationId) + if (!observation) { + console.warn(`observation ${reference.observationId} not found while processing pending attachment ${reference.attachmentId}`) + return + } + const attachment = observation.attachmentFor(reference.attachmentId) + if (!attachment || !attachment.stagedContentId) { + console.warn(`attachment ${reference.attachmentId} not found or has no staged content on observation ${reference.observationId}`) + return + } + const stagedContentId = attachment.stagedContentId + const stagedPath = await attachmentStore.stagedContentPath(stagedContentId) + if (stagedPath instanceof AttachmentStoreError) { + console.error(`error resolving staged content path for attachment ${attachment.id}`, stagedPath) + return + } + const outcome = await runPipeline(attachmentHooks, attachment, stagedPath) + if (outcome.outcome === 'pass') { + const finalized = await attachmentStore.saveContent(new StagedAttachmentContentRef(stagedContentId), attachment.id, observation) + if (finalized instanceof AttachmentStoreError) { + console.error(`error finalizing clean attachment ${attachment.id}`, finalized) + return + } + const patch: AttachmentPatchAttrs = { ...(finalized || {}), processingStatus: 'clean', stagedContentId: undefined } + await obsRepo.patchAttachment(observation, attachment.id, patch) + return + } + if (outcome.outcome === 'reject') { + await attachmentStore.deleteStagedContent(stagedContentId) + await obsRepo.patchAttachment(observation, attachment.id, { + processingStatus: 'rejected', + processingMessage: outcome.reason, + processingHook: outcome.hookName, + stagedContentId: undefined + }) + return + } + // outcome.outcome === 'error' + const retryCount = (attachment.processingRetryCount || 0) + 1 + if (retryCount < retryLimit) { + await obsRepo.patchAttachment(observation, attachment.id, { + processingStatus: 'pending', + processingMessage: outcome.error.message, + processingHook: outcome.hookName, + processingRetryCount: retryCount + }) + return + } + await attachmentStore.deleteStagedContent(stagedContentId) + await obsRepo.patchAttachment(observation, attachment.id, { + processingStatus: 'error', + processingMessage: outcome.error.message, + processingHook: outcome.hookName, + processingRetryCount: retryCount, + stagedContentId: undefined + }) + } + catch (err) { + console.error(`unexpected error processing pending attachment ${reference.attachmentId} on observation ${reference.observationId}`, err) + } +} + +export type AttachmentProcessingJob = { + stop: () => void +} + +/** + * Starts the core-owned background job that finds attachments staged by + * storeAttachmentContent and runs them through whatever attachment- + * processing hooks plugins have registered. Modeled on the image plugin's + * self-rescheduling setTimeout shape, but core-owned rather than plugin- + * owned, since it must run hooks contributed by any enabled plugin. + */ +export function startAttachmentProcessing( + obsRepoForEvent: ObservationRepositoryForEvent, + attachmentStore: AttachmentStore, + attachmentHooks: AttachmentHook[], + console: Console, + config: Partial = {} +): AttachmentProcessingJob { + const resolvedConfig: AttachmentProcessingConfig = { ...defaultAttachmentProcessingConfig, ...config } + let stopped = false + + async function processNextBatch(): Promise { + if (stopped) { + return + } + try { + const references = await findPendingAttachments(resolvedConfig.batchSize) + for (const reference of references) { + if (stopped) { + break + } + await processPendingAttachment(reference, obsRepoForEvent, attachmentStore, attachmentHooks, resolvedConfig.retryLimit, console) + } + } + catch (err) { + console.error('error processing pending attachments', err) + } + if (!stopped) { + setTimeout(() => { processNextBatch() }, resolvedConfig.intervalSeconds * 1000) + } + } + + processNextBatch() + + return { + stop: () => { + stopped = true + } + } +} diff --git a/service/src/main.impl/main.impl.plugins.ts b/service/src/main.impl/main.impl.plugins.ts index e0f2411c4..dd603dc93 100644 --- a/service/src/main.impl/main.impl.plugins.ts +++ b/service/src/main.impl/main.impl.plugins.ts @@ -6,8 +6,10 @@ import { FeedServiceTypeRepositoryToken, FeedsPluginHooks } from '../plugins.api import { IconPluginHooks, StaticIconRepositoryToken } from '../plugins.api/plugins.api.icons' import { MageEventsPluginHooks } from '../plugins.api/plugins.api.events' import { WebRoutesHooks } from '../plugins.api/plugins.api.web' +import { AttachmentHook, AttachmentProcessingPluginHooks } from '../plugins.api/plugins.api.attachments' +import { loadAttachmentHooks } from './plugin_hooks/main.impl.plugin_hooks.attachments' -export type PluginHooks = MageEventsPluginHooks & FeedsPluginHooks & IconPluginHooks & WebRoutesHooks +export type PluginHooks = MageEventsPluginHooks & FeedsPluginHooks & IconPluginHooks & WebRoutesHooks & AttachmentProcessingPluginHooks export interface InjectableServices { (token: InjectionToken): Service @@ -15,11 +17,15 @@ export interface InjectableServices { export type AddPluginWebRoutes = (pluginId: string, webRoutes: WebRoutesHooks) => void +// Export definition for AttachmentHooks +export type AddPluginAttachmentHooks = (pluginId: string, hooks: AttachmentHook[]) => void + export async function integratePluginHooks( pluginId: string, plugin: InitPluginHook, injectService: InjectableServices, addWebRoutesFromPlugin: AddPluginWebRoutes, + collectAttachmentHooks: AddPluginAttachmentHooks, ): Promise { let injection: Injection | null = null let hooks: PluginHooks @@ -36,6 +42,7 @@ export async function integratePluginHooks( await loadMageEventsHoooks(pluginId, hooks) await loadIconsHooks(pluginId, hooks, injectService(StaticIconRepositoryToken)) await loadFeedsHooks(pluginId, hooks, injectService(FeedServiceTypeRepositoryToken)) + await loadAttachmentHooks(pluginId, hooks, collectAttachmentHooks) if (hooks.webRoutes) { await addWebRoutesFromPlugin(pluginId, hooks) } diff --git a/service/src/main.impl/plugin_hooks/main.impl.plugin_hooks.attachments.ts b/service/src/main.impl/plugin_hooks/main.impl.plugin_hooks.attachments.ts new file mode 100644 index 000000000..d80739c36 --- /dev/null +++ b/service/src/main.impl/plugin_hooks/main.impl.plugin_hooks.attachments.ts @@ -0,0 +1,15 @@ +// Imports +import { AttachmentProcessingPluginHooks } from "../../plugins.api/plugins.api.attachments"; +import { AddPluginAttachmentHooks } from "../main.impl.plugins"; + +// Collects a single plugin's contributed attachment-processing hooks (if any) and hands them off to be stored in the shared in-memory hook registry. +export async function loadAttachmentHooks(moduleName: string, hooks: Partial, collectHooks: AddPluginAttachmentHooks): Promise { + // Array of hooks & guard + const attachmentHooks = hooks.attachmentHooks + if (!attachmentHooks){ + return + } + + // Call collect hooks + collectHooks(moduleName, attachmentHooks) +} diff --git a/service/src/models/observation.js b/service/src/models/observation.js index 511f61cd1..1e75c9d46 100644 --- a/service/src/models/observation.js +++ b/service/src/models/observation.js @@ -41,7 +41,13 @@ const AttachmentSchema = new Schema({ width: { type: Number, required: false }, height: { type: Number, required: false }, oriented: { type: Boolean, required: true, default: false }, - thumbnails: [ThumbnailSchema] + thumbnails: [ThumbnailSchema], + processingStatus: { type: String, enum: ['pending', 'clean', 'rejected', 'error'], required: false }, + processingMessage: { type: String, required: false }, + processingHook: { type: String, required: false }, + stagedContentId: { type: String, required: false }, + processingRetryCount: { type: Number, required: false, default: 0 } + }, { strict: false }); diff --git a/service/src/plugins.api/plugins.api.attachments.ts b/service/src/plugins.api/plugins.api.attachments.ts new file mode 100644 index 000000000..c016a99e8 --- /dev/null +++ b/service/src/plugins.api/plugins.api.attachments.ts @@ -0,0 +1,31 @@ +import { Attachment } from '../entities/observations/entities.observations' + +// Define the outcomes from the attachment +export type AttachmentHookOutcome = | { outcome: 'pass' } | { outcome: 'reject', reason: string } | { outcome: 'error', error: Error } + +// Defining the signature for future hooks +export type AttachmentHook = (attachment: Attachment, stagedFilePath: string) => Promise +export type AttachmentPipelineResult = AttachmentHookOutcome & { hookName?: string } + +// Defining the interface +export interface AttachmentProcessingPluginHooks { + attachmentHooks: AttachmentHook[] +} +export async function runPipeline(hooks: AttachmentHook[], attachment: Attachment, stagedFilePath: string): Promise { + for (const hook of hooks) { + + try { + const outcome = await hook(attachment, stagedFilePath) + + // Check if outcome is problematic + if (outcome.outcome === 'reject' || outcome.outcome === 'error') { + + // Match the spread pattern + return { ...outcome, hookName: hook.name } + } + } catch (err) { + return {outcome: 'error', error: err instanceof Error ? err : new Error(String(err)), hookName: hook.name} + } + } + return {outcome: 'pass'} +} \ No newline at end of file diff --git a/service/test/app/observations/app.observations.test.ts b/service/test/app/observations/app.observations.test.ts index 8572c64c1..14c0d931f 100644 --- a/service/test/app/observations/app.observations.test.ts +++ b/service/test/app/observations/app.observations.test.ts @@ -1614,7 +1614,7 @@ describe('observations use case interactions', function() { } ] obs = Observation.evaluate(baseObsAttrs, mageEvent) - storeAttachmentContent = StoreAttachmentContent(permissions, store) + storeAttachmentContent = StoreAttachmentContent(permissions, store, []) expect(obs.validation.hasErrors).to.be.false }) diff --git a/service/test/main/main.plugins.test.ts b/service/test/main/main.plugins.test.ts index 906368109..f4ecf92fe 100644 --- a/service/test/main/main.plugins.test.ts +++ b/service/test/main/main.plugins.test.ts @@ -6,9 +6,10 @@ import { Arg, Substitute as Sub, SubstituteOf } from '@fluffy-spoon/substitute' import * as plugins from '../../lib/main.impl/main.impl.plugins' import { InitPluginHook, InjectionToken } from '../../lib/plugins.api' import { WebRoutesHooks } from '../../lib/plugins.api/plugins.api.web' -import { AddPluginWebRoutes } from '../../lib/main.impl/main.impl.plugins' +import { AddPluginWebRoutes, AddPluginAttachmentHooks } from '../../lib/main.impl/main.impl.plugins' import { AppRequestContext } from '../../lib/app.api/app.api.global' import { UserExpanded } from '../../lib/entities/users/entities.users' +import { AttachmentHook } from '../../lib/plugins.api/plugins.api.attachments' interface Service1 {} interface Service2 {} @@ -19,6 +20,7 @@ class Service2Impl implements Service2 {} const serviceMap = new Map([[ Service1Token, new Service1Impl() ], [ Service2Token, new Service2Impl() ]]) const injectService: plugins.InjectableServices = (token: any) => serviceMap.get(token) as any const initPluginRoutes: AddPluginWebRoutes = (pluginId: string, initPluginRoutes: WebRoutesHooks) => void(0) +const collectAttachmentHooks: AddPluginAttachmentHooks = (pluginId: string, hooks) => void(0) interface InjectServiceHandle { injectService: typeof injectService @@ -28,14 +30,20 @@ interface InitPluginRoutesHandle { initPluginRoutes: typeof initPluginRoutes } +interface CollectAttachmentHooksHandle { + collectAttachmentHooks: typeof collectAttachmentHooks +} + describe('loading plugins', function() { let mockInjectService: SubstituteOf let mockInitPluginRoutes: SubstituteOf + let mockCollectAttachmentHooks: SubstituteOf beforeEach(function() { mockInjectService = Sub.for() mockInitPluginRoutes = Sub.for() + mockCollectAttachmentHooks = Sub.for() }) it('runs the init hook with requested injected serivces', async function() { @@ -57,7 +65,7 @@ describe('loading plugins', function() { } } initPlugin.inject = injectRequest - await plugins.integratePluginHooks(pluginId, initPlugin, injectService, initPluginRoutes) + await plugins.integratePluginHooks(pluginId, initPlugin, injectService, initPluginRoutes, collectAttachmentHooks) expect(injected).to.have.property('service1').instanceOf(Service1Impl) expect(injected).to.have.property('service2').instanceOf(Service2Impl) @@ -90,9 +98,54 @@ describe('loading plugins', function() { } initPlugin.inject = injectRequest mockInitPluginRoutes.initPluginRoutes(Arg.all()).mimicks(initPluginRoutes) - await plugins.integratePluginHooks(pluginId, initPlugin, injectService, mockInitPluginRoutes.initPluginRoutes) + await plugins.integratePluginHooks(pluginId, initPlugin, injectService, mockInitPluginRoutes.initPluginRoutes, collectAttachmentHooks) mockInitPluginRoutes.received(1).initPluginRoutes(Arg.all()) mockInitPluginRoutes.received(1).initPluginRoutes(pluginId, hook) }) + + it('collects attachment hooks for plugin when provided', async function() { + + const pluginId = '@testing/test3' + const injectRequest = {} + const exampleHook: AttachmentHook = async (attachment, stagedFilePath) => ({ outcome: 'pass' }) + const hook = { + attachmentHooks: [ exampleHook ] + } + + const initPlugin: InitPluginHook = { + inject: { + + }, + init: async (services) => { + return hook + } + } + initPlugin.inject = injectRequest + mockCollectAttachmentHooks.collectAttachmentHooks(Arg.all()).mimicks(collectAttachmentHooks) + await plugins.integratePluginHooks(pluginId, initPlugin, injectService, initPluginRoutes, mockCollectAttachmentHooks.collectAttachmentHooks) + + mockCollectAttachmentHooks.received(1).collectAttachmentHooks(Arg.all()) + mockCollectAttachmentHooks.received(1).collectAttachmentHooks(pluginId, hook.attachmentHooks) + }) + + it('does not collect attachment hooks when plugin provides none', async function() { + + const pluginId = '@testing/test4' + const injectRequest = {} + + const initPlugin: InitPluginHook = { + inject: { + + }, + init: async (services) => { + return {} + } + } + initPlugin.inject = injectRequest + mockCollectAttachmentHooks.collectAttachmentHooks(Arg.all()).mimicks(collectAttachmentHooks) + await plugins.integratePluginHooks(pluginId, initPlugin, injectService, initPluginRoutes, mockCollectAttachmentHooks.collectAttachmentHooks) + + mockCollectAttachmentHooks.didNotReceive().collectAttachmentHooks(Arg.all()) + }) }) \ No newline at end of file diff --git a/web-app/src/app/filter/filter.types.ts b/web-app/src/app/filter/filter.types.ts index a367feb17..a3dea663f 100644 --- a/web-app/src/app/filter/filter.types.ts +++ b/web-app/src/app/filter/filter.types.ts @@ -1,5 +1,6 @@ import { User } from "@ngageoint/mage.web-core-lib/user"; import { filterChanges } from "../event/event.types"; +import { AttachmentAction } from "../observation/observation-edit/observation-edit-attachment/observation-edit-attachment-action"; export type FormField = { name: string; @@ -103,7 +104,15 @@ export type Attachment = { oriented: boolean; relativePath: string; size: number; - url: string; + url?: string; // Should have '?' + processingStatus?: 'clean' | 'rejected' | 'pending' | 'error'; // Union + processingMessage?: string + processingHook?: string + action?: AttachmentAction + // TODO: never actually assigned anywhere in the codebase - appears to be + // vestigial/dead code from an incomplete cache-busting feature. Left + // loosely typed rather than guessing at an intended shape. + synced?: any }; export type Observation = { diff --git a/web-app/src/app/observation/attachment/attachment.component.html b/web-app/src/app/observation/attachment/attachment.component.html index 4305ab5b4..474a9b712 100644 --- a/web-app/src/app/observation/attachment/attachment.component.html +++ b/web-app/src/app/observation/attachment/attachment.component.html @@ -30,7 +30,7 @@
- +
@@ -57,7 +57,22 @@
- +
+ + + +
+ error +
{{attachment.name | filename:12}}
+
Upload Failed
+
Tap for details
+
+ +
+
diff --git a/web-app/src/app/observation/attachment/attachment.component.scss b/web-app/src/app/observation/attachment/attachment.component.scss index 4ee6267db..6e72a0488 100644 --- a/web-app/src/app/observation/attachment/attachment.component.scss +++ b/web-app/src/app/observation/attachment/attachment.component.scss @@ -34,6 +34,15 @@ margin-bottom: 8px; } +.media-unknown--clickable { + cursor: pointer; +} + +.media-unknown__hint { + font-size: 11px; + opacity: 0.75; +} + .media-delete__mask { opacity: 0.5; } diff --git a/web-app/src/app/observation/attachment/attachment.component.ts b/web-app/src/app/observation/attachment/attachment.component.ts index d262f6e0a..3307d6813 100644 --- a/web-app/src/app/observation/attachment/attachment.component.ts +++ b/web-app/src/app/observation/attachment/attachment.component.ts @@ -1,6 +1,8 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; +import { MatSnackBar } from '@angular/material/snack-bar'; import { AttachmentAction } from '../observation-edit/observation-edit-attachment/observation-edit-attachment-action'; import { SessionService } from 'mage-web-app/http/session.service'; +import { Attachment } from '../../filter/filter.types' @Component({ selector: 'observation-attachment', @@ -9,7 +11,7 @@ import { SessionService } from 'mage-web-app/http/session.service'; standalone: false }) export class AttachmentComponent implements OnInit { - @Input() attachment: any + @Input() attachment: Attachment @Input() clickable: boolean @Input() edit: boolean @Input() label: string | boolean @@ -30,7 +32,8 @@ export class AttachmentComponent implements OnInit { actions: typeof AttachmentAction = AttachmentAction constructor( - private sessionService: SessionService + private sessionService: SessionService, + private snackBar: MatSnackBar ) { this.token = sessionService.getToken() } @@ -53,4 +56,12 @@ export class AttachmentComponent implements OnInit { return mimeType ? mimeType : '' } } + + isFailed(): boolean { + return this.attachment.processingStatus === 'rejected' || this.attachment.processingStatus === 'error' + } + + showFailureMessage(): void { + this.snackBar.open(this.attachment.processingMessage, 'Dismiss', { duration: 6000 }) + } } diff --git a/web-app/src/app/observation/observation-edit/observation-edit-attachment/observation-edit-attachment.component.ts b/web-app/src/app/observation/observation-edit/observation-edit-attachment/observation-edit-attachment.component.ts index 0e136a2d0..676ef1cc6 100644 --- a/web-app/src/app/observation/observation-edit/observation-edit-attachment/observation-edit-attachment.component.ts +++ b/web-app/src/app/observation/observation-edit/observation-edit-attachment/observation-edit-attachment.component.ts @@ -39,8 +39,7 @@ export class ObservationEditAttachmentComponent implements OnInit { allAttachments(): any[] { const observationFormId = this.formGroup.get('id')?.value const attachments = (this.attachments || []).filter(attachment => { - return attachment.url && - attachment.observationFormId === observationFormId && + return attachment.observationFormId === observationFormId && attachment.fieldName === this.definition.name }); diff --git a/web-app/src/app/observation/observation-list/observation-list-item.component.html b/web-app/src/app/observation/observation-list/observation-list-item.component.html index 7790355de..a5ab130f8 100644 --- a/web-app/src/app/observation/observation-list/observation-list-item.component.html +++ b/web-app/src/app/observation/observation-list/observation-list-item.component.html @@ -51,12 +51,16 @@ -
+
+ +
+ {{ failedAttachmentCount() > 1 ? 'Attachments Failed - ' + failedAttachmentCount() : 'Attachment Failed' }} +
diff --git a/web-app/src/app/observation/observation-list/observation-list-item.component.scss b/web-app/src/app/observation/observation-list/observation-list-item.component.scss index c59098b80..f487adf73 100644 --- a/web-app/src/app/observation/observation-list/observation-list-item.component.scss +++ b/web-app/src/app/observation/observation-list/observation-list-item.component.scss @@ -93,6 +93,21 @@ mat-card-actions { display: block; } +.item-media-chip { + margin-top: 8px; +} + +.item-media-chip-label { + display: inline-flex; + align-items: center; + padding: 2px 8px; + border-radius: 12px; + font-size: 12px; + font-weight: 500; + background: #ffebee; + color: #c62828; +} + .item-important { padding: 8px 0; display: flex; diff --git a/web-app/src/app/observation/observation-list/observation-list-item.component.ts b/web-app/src/app/observation/observation-list/observation-list-item.component.ts index f66f9490d..e48d4c536 100644 --- a/web-app/src/app/observation/observation-list/observation-list-item.component.ts +++ b/web-app/src/app/observation/observation-list/observation-list-item.component.ts @@ -206,4 +206,17 @@ export class ObservationListItemComponent implements OnChanges { updateFavorites(): void { this.favorites = this.observation.favoriteUserIds.length } + + // If observation shows multiple attachments, always show the 'passed' as the thumbnail but ensure a flag is also shown for the failed + representativeAttachment(): any { + return this.attachments.find(attachment => attachment.processingStatus !== 'rejected' && attachment.processingStatus !== 'error') || this.attachments[0] + } + + hasFailedAttachment(): boolean { + return this.attachments.some(attachment => attachment.processingStatus === 'rejected' || attachment.processingStatus === 'error') + } + + failedAttachmentCount(): number { + return this.attachments.filter(attachment => attachment.processingStatus === 'rejected' || attachment.processingStatus === 'error').length + } }