From 72f91c55eab151cefa070ad672bdd33bcf9300e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Dec 2024 00:59:54 +0100 Subject: [PATCH 1/3] Add anchor impressions support with database schema and API integration --- .../migrations/15_anchor_impressions.sql | 37 +++ db_schema/schema.sql | 2 +- fixtures/anchorImpressions.json | 202 ++++++++++++++++ src/api/connectors/AnchorConnector.ts | 10 + src/db/AnchorRepository.ts | 87 +++++++ src/db/DBInitializer.ts | 3 + src/schema/anchor/impressions.json | 225 ++++++++++++++++++ src/types/provider/anchor.ts | 45 ++++ tests/api_e2e/anchor.test.js | 11 + 9 files changed, 621 insertions(+), 1 deletion(-) create mode 100644 db_schema/migrations/15_anchor_impressions.sql create mode 100644 fixtures/anchorImpressions.json create mode 100644 src/schema/anchor/impressions.json diff --git a/db_schema/migrations/15_anchor_impressions.sql b/db_schema/migrations/15_anchor_impressions.sql new file mode 100644 index 00000000..3077e776 --- /dev/null +++ b/db_schema/migrations/15_anchor_impressions.sql @@ -0,0 +1,37 @@ +-- Create table for storing aggregate impression metrics +CREATE TABLE IF NOT EXISTS anchorImpressions ( + account_id INTEGER NOT NULL, + date_start DATE NOT NULL, + date_end DATE NOT NULL, + total_impressions INTEGER NOT NULL, + total_considerations INTEGER NOT NULL, + total_streams INTEGER NOT NULL, + considerations_conversion_rate DECIMAL(10,5), + streams_conversion_rate DECIMAL(10,5), + PRIMARY KEY (account_id, date_start, date_end) +); + +-- Create table for daily impression data +CREATE TABLE IF NOT EXISTS anchorImpressionsDaily ( + account_id INTEGER NOT NULL, + date DATE NOT NULL, + impressions INTEGER NOT NULL, + PRIMARY KEY (account_id, date) +); + +-- Create table for impression sources breakdown +CREATE TABLE IF NOT EXISTS anchorImpressionsSources ( + account_id INTEGER NOT NULL, + date_start DATE NOT NULL, + date_end DATE NOT NULL, + source_id VARCHAR(32) NOT NULL, + source_name VARCHAR(64) NOT NULL, + impression_count INTEGER NOT NULL, + PRIMARY KEY (account_id, date_start, date_end, source_id) +); + +-- Add index for querying date ranges efficiently on sources table +CREATE INDEX idx_impression_sources_date ON anchorImpressionsSources(account_id, date_start, date_end); + +-- Record the migration +INSERT INTO migrations (migration_id, migration_name) VALUES (15, 'anchor impressions'); \ No newline at end of file diff --git a/db_schema/schema.sql b/db_schema/schema.sql index 1a84d0bb..6f9da470 100644 --- a/db_schema/schema.sql +++ b/db_schema/schema.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS migrations ( -- ----------------------------------------- -- IMPORTANT: this is the schema version -- ID has to be incremented for each change -INSERT INTO migrations (migration_id, migration_name) VALUES (14, 'genericHoster'); +INSERT INTO migrations (migration_id, migration_name) VALUES (15, 'anchor impressions'); -- ----------------------------------------- CREATE TABLE IF NOT EXISTS events ( diff --git a/fixtures/anchorImpressions.json b/fixtures/anchorImpressions.json new file mode 100644 index 00000000..d7ecd45f --- /dev/null +++ b/fixtures/anchorImpressions.json @@ -0,0 +1,202 @@ +{ + "provider": "anchor", + "version": 1, + "retrieved": "2023-05-05T16:20:56.696026", + "meta": { + "show": "123abcde", + "endpoint": "impressions" + }, + "range": { + "start": "2023-05-01", + "end": "2023-05-04" + }, + "data": { + "impressionsFunnel": { + "data": { + "value": { + "counts": [ + { + "id": "impressions", + "count": 4571 + }, + { + "id": "considerations", + "count": 731, + "conversionPercent": 0.15992 + }, + { + "id": "streams", + "count": 381, + "conversionPercent": 0.5212 + } + ] + }, + "isDistributedToSpotify": true + }, + "error": null + }, + "impressions": { + "data": { + "value": 30868, + "isDistributedToSpotify": true + }, + "error": null + }, + "dailyImpressions": { + "data": { + "value": [ + { + "date": 1732233600, + "value": 1145 + }, + { + "date": 1732320000, + "value": 860 + }, + { + "date": 1732406400, + "value": 846 + }, + { + "date": 1732492800, + "value": 952 + }, + { + "date": 1732579200, + "value": 1054 + }, + { + "date": 1732665600, + "value": 1079 + }, + { + "date": 1732752000, + "value": 915 + }, + { + "date": 1732838400, + "value": 1047 + }, + { + "date": 1732924800, + "value": 974 + }, + { + "date": 1733011200, + "value": 1050 + }, + { + "date": 1733097600, + "value": 849 + }, + { + "date": 1733184000, + "value": 873 + }, + { + "date": 1733270400, + "value": 1449 + }, + { + "date": 1733356800, + "value": 983 + }, + { + "date": 1733443200, + "value": 1036 + }, + { + "date": 1733529600, + "value": 857 + }, + { + "date": 1733616000, + "value": 955 + }, + { + "date": 1733702400, + "value": 1115 + }, + { + "date": 1733788800, + "value": 1261 + }, + { + "date": 1733875200, + "value": 1100 + }, + { + "date": 1733961600, + "value": 979 + }, + { + "date": 1734048000, + "value": 1381 + }, + { + "date": 1734134400, + "value": 882 + }, + { + "date": 1734220800, + "value": 1335 + }, + { + "date": 1734307200, + "value": 956 + }, + { + "date": 1734393600, + "value": 1146 + }, + { + "date": 1734480000, + "value": 967 + }, + { + "date": 1734566400, + "value": 932 + }, + { + "date": 1734652800, + "value": 1000 + }, + { + "date": 1734739200, + "value": 890 + } + ], + "isDistributedToSpotify": true + }, + "error": null + }, + "impressionsBySource": { + "data": { + "value": [ + { + "id": "home", + "value": 9082, + "displayName": "Spotify Home" + }, + { + "id": "search", + "value": 20849, + "displayName": "Spotify Search" + }, + { + "id": "library", + "value": 936, + "displayName": "Spotify Library" + }, + { + "id": "other", + "value": 1, + "displayName": "Other Spotify features" + } + ], + "isDistributedToSpotify": true + }, + "error": null + } + } +} \ No newline at end of file diff --git a/src/api/connectors/AnchorConnector.ts b/src/api/connectors/AnchorConnector.ts index 58402f27..27abe6e0 100644 --- a/src/api/connectors/AnchorConnector.ts +++ b/src/api/connectors/AnchorConnector.ts @@ -17,6 +17,7 @@ import podcastEpisodeSchema from '../../schema/anchor/podcastEpisode.json' import totalPlaysSchema from '../../schema/anchor/totalPlays.json' import totalPlaysByEpisodeSchema from '../../schema/anchor/totalPlaysByEpisode.json' import uniqueListenersSchema from '../../schema/anchor/uniqueListeners.json' +import impressionsSchema from '../../schema/anchor/impressions.json' import { ConnectorPayload } from '../../types/connector' import { @@ -37,6 +38,7 @@ import { RawAnchorTotalPlaysByEpisodeData, RawAnchorUniqueListenersData, RawAnchorEpisodesPageData, + RawAnchorImpressionData, } from '../../types/provider/anchor' import { AnchorRepository } from '../../db/AnchorRepository' import { isArray } from 'mathjs' @@ -254,6 +256,14 @@ class AnchorConnector implements ConnectorHandler { accountId, payload.data.data as RawAnchorUniqueListenersData ) + } else if (endpoint == 'impressions') { + validateJsonApiPayload(impressionsSchema, rawPayload) + await this.repo.storeImpressions( + accountId, + rawPayload.range.start, + rawPayload.range.end, + payload.data as RawAnchorImpressionData + ) } else { throw new PayloadError( `Unknown endpoint in meta: ${rawPayload.meta.endpoint}` diff --git a/src/db/AnchorRepository.ts b/src/db/AnchorRepository.ts index dcab5690..d5ba0b43 100644 --- a/src/db/AnchorRepository.ts +++ b/src/db/AnchorRepository.ts @@ -18,6 +18,7 @@ import { RawAnchorTotalPlaysByEpisodeData, RawAnchorUniqueListenersData, RawAnchorEpisodesPageData, + RawAnchorImpressionData, } from '../types/provider/anchor' const getDateDBString = (date: Date): string => { @@ -511,6 +512,92 @@ class AnchorRepository { return queryPromise } + + async storeImpressions( + accountId: number, + startDate: string, + endDate: string, + data: RawAnchorImpressionData + ): Promise { + const promises: Promise[] = [] + + // Store aggregate impression data + if (data.impressions?.data?.value) { + const replaceStmt = `REPLACE INTO anchorImpressions ( + account_id, + date_start, + date_end, + total_impressions, + total_considerations, + total_streams, + considerations_conversion_rate, + streams_conversion_rate + ) VALUES (?,?,?,?,?,?,?,?)` + + const funnel = data.impressionsFunnel.data.value.counts + const considerations = funnel.find((x) => x.id === 'considerations') + const streams = funnel.find((x) => x.id === 'streams') + + promises.push( + this.pool.query(replaceStmt, [ + accountId, + startDate, + endDate, + data.impressions.data.value, + considerations?.count || 0, + streams?.count || 0, + considerations?.conversionPercent || 0, + streams?.conversionPercent || 0, + ]) + ) + } + + // Store daily impressions + if (data.dailyImpressions?.data?.value) { + const dailyStmt = `REPLACE INTO anchorImpressionsDaily ( + account_id, + date, + impressions + ) VALUES (?,?,?)` + + data.dailyImpressions.data.value.forEach((daily) => { + promises.push( + this.pool.query(dailyStmt, [ + accountId, + getDateFromTimestamp(daily.date), + daily.value, + ]) + ) + }) + } + + // Store impression sources + if (data.impressionsBySource?.data?.value) { + const sourceStmt = `REPLACE INTO anchorImpressionsSources ( + account_id, + date_start, + date_end, + source_id, + source_name, + impression_count + ) VALUES (?,?,?,?,?,?)` + + data.impressionsBySource.data.value.forEach((source) => { + promises.push( + this.pool.query(sourceStmt, [ + accountId, + startDate, + endDate, + source.id, + source.displayName, + source.value, + ]) + ) + }) + } + + return Promise.all(promises) + } } export { AnchorRepository } diff --git a/src/db/DBInitializer.ts b/src/db/DBInitializer.ts index be92b6ce..8b2f0a48 100644 --- a/src/db/DBInitializer.ts +++ b/src/db/DBInitializer.ts @@ -133,6 +133,9 @@ class DBInitializer { } const migrationIdGoal = this.getMigrationGoal() + console.log('Latest migration id:', latestMigrationId) + console.log('Migration id goal:', migrationIdGoal) + if (latestMigrationId >= migrationIdGoal) { return } diff --git a/src/schema/anchor/impressions.json b/src/schema/anchor/impressions.json new file mode 100644 index 00000000..3808639d --- /dev/null +++ b/src/schema/anchor/impressions.json @@ -0,0 +1,225 @@ +{ + "$id": "http://openpodcast.dev/schema/anchor/impressions.schema.json", + "title": "Anchor / Spotify for Creators - Impressions data", + "type": "object", + "properties": { + "provider": { + "type": "string" + }, + "version": { + "type": "number" + }, + "retrieved": { + "type": "string" + }, + "meta": { + "type": "object", + "properties": { + "show": { + "type": "string" + }, + "endpoint": { + "type": "string" + } + }, + "required": [ + "show", + "endpoint" + ] + }, + "range": { + "type": "object", + "properties": { + "start": { + "type": "string" + }, + "end": { + "type": "string" + } + }, + "required": [ + "start", + "end" + ] + }, + "data": { + "type": "object", + "properties": { + "impressionsFunnel": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "value": { + "type": "object", + "properties": { + "counts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "count": { + "type": "number" + }, + "conversionPercent": { + "type": "number" + } + }, + "required": [ + "id", + "count" + ] + } + } + }, + "required": [ + "counts" + ] + }, + "isDistributedToSpotify": { + "type": "boolean" + } + }, + "required": [ + "value", + "isDistributedToSpotify" + ] + }, + "error": {} + }, + "required": [ + "data", + "error" + ] + }, + "impressions": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "isDistributedToSpotify": { + "type": "boolean" + } + }, + "required": [ + "value", + "isDistributedToSpotify" + ] + }, + "error": {} + }, + "required": [ + "data", + "error" + ] + }, + "dailyImpressions": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "value": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "number" + }, + "value": { + "type": "number" + } + }, + "required": [ + "date", + "value" + ] + } + }, + "isDistributedToSpotify": { + "type": "boolean" + } + }, + "required": [ + "value", + "isDistributedToSpotify" + ] + }, + "error": {} + }, + "required": [ + "data", + "error" + ] + }, + "impressionsBySource": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "value": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": "number" + }, + "displayName": { + "type": "string" + } + }, + "required": [ + "id", + "value", + "displayName" + ] + } + }, + "isDistributedToSpotify": { + "type": "boolean" + } + }, + "required": [ + "value", + "isDistributedToSpotify" + ] + }, + "error": {} + }, + "required": [ + "data", + "error" + ] + } + }, + "required": [ + "impressionsFunnel", + "impressions", + "dailyImpressions", + "impressionsBySource" + ] + } + }, + "required": [ + "provider", + "version", + "retrieved", + "meta", + "range", + "data" + ] +} \ No newline at end of file diff --git a/src/types/provider/anchor.ts b/src/types/provider/anchor.ts index 0a62c578..57cb503b 100644 --- a/src/types/provider/anchor.ts +++ b/src/types/provider/anchor.ts @@ -185,6 +185,50 @@ export interface RawAnchorUniqueListenersData { columnHeaders: [AnchorColumnHeader] } +export interface RawAnchorImpressionData { + impressionsFunnel: { + data: { + value: { + counts: Array<{ + id: string + count: number + conversionPercent?: number + }> + } + isDistributedToSpotify: boolean + } + error: null | any + } + impressions: { + data: { + value: number + isDistributedToSpotify: boolean + } + error: null | any + } + dailyImpressions: { + data: { + value: Array<{ + date: number + value: number + }> + isDistributedToSpotify: boolean + } + error: null | any + } + impressionsBySource: { + data: { + value: Array<{ + id: string + value: number + displayName: string + }> + isDistributedToSpotify: boolean + } + error: null | any + } +} + export type AnchorDataPayload = | { kind: 'aggregatedPerformance'; data: AnchorAggregatedPerformanceData } | { kind: 'audienceSize'; data: RawAnchorAudienceSizeData } @@ -203,6 +247,7 @@ export type AnchorDataPayload = | { kind: 'totalPlays'; data: RawAnchorTotalPlaysData } | { kind: 'totalPlaysByEpisode'; data: RawAnchorTotalPlaysByEpisodeData } | { kind: 'uniqueListeners'; data: RawAnchorUniqueListenersData } + | { kind: 'impressions'; data: RawAnchorImpressionData } export interface AnchorConnectorPayload { meta: { diff --git a/tests/api_e2e/anchor.test.js b/tests/api_e2e/anchor.test.js index a1d6eae3..57460c49 100644 --- a/tests/api_e2e/anchor.test.js +++ b/tests/api_e2e/anchor.test.js @@ -16,6 +16,7 @@ const anchorTotalPlaysPayload = require('../../fixtures/anchorTotalPlays.json') const anchorTotalPlaysByEpisodePayload = require('../../fixtures/anchorTotalPlaysByEpisode.json') const anchorUniqueListenersPayload = require('../../fixtures/anchorUniqueListeners.json') const anchorEpisodesPagePayload = require('../../fixtures/anchorEpisodesPage.json') +const anchorImpressionsPayload = require('../../fixtures/anchorImpressions.json') const auth = require('./authheader') @@ -229,3 +230,13 @@ describe('check Connector API with anchorUniqueListenersPayload', () => { expect(response.statusCode).toBe(200) }) }) + +describe('check Connector API with anchorImpressionsPayload', () => { + it('should return status 200 when sending proper Anchor payload', async () => { + const response = await request(baseURL) + .post('/connector') + .set(auth) + .send(anchorImpressionsPayload) + expect(response.statusCode).toBe(200) + }) +}) From a2b8f13b916dffc3c15ff7a6db66c1daf41569b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 13 Aug 2025 14:31:31 +0200 Subject: [PATCH 2/3] Update schema --- db_schema/migrations/15_anchor_impressions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_schema/migrations/15_anchor_impressions.sql b/db_schema/migrations/15_anchor_impressions.sql index 3077e776..432b65ef 100644 --- a/db_schema/migrations/15_anchor_impressions.sql +++ b/db_schema/migrations/15_anchor_impressions.sql @@ -34,4 +34,4 @@ CREATE TABLE IF NOT EXISTS anchorImpressionsSources ( CREATE INDEX idx_impression_sources_date ON anchorImpressionsSources(account_id, date_start, date_end); -- Record the migration -INSERT INTO migrations (migration_id, migration_name) VALUES (15, 'anchor impressions'); \ No newline at end of file +INSERT INTO migrations (migration_id, migration_name) VALUES (15, 'anchor impressions'); From b5ea317f22191fdeed356592d8b0d09b7c5b1073 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 13 Aug 2025 15:41:39 +0200 Subject: [PATCH 3/3] Fix TypeScript build errors - Add null check for rawPayload.range in impressions endpoint - Fix type conversion by using correct payload.data.data structure - Exclude test files from TypeScript build to prevent Jest type errors --- src/api/connectors/AnchorConnector.ts | 8 +++++++- tsconfig.json | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/api/connectors/AnchorConnector.ts b/src/api/connectors/AnchorConnector.ts index 27abe6e0..1210d12e 100644 --- a/src/api/connectors/AnchorConnector.ts +++ b/src/api/connectors/AnchorConnector.ts @@ -258,11 +258,17 @@ class AnchorConnector implements ConnectorHandler { ) } else if (endpoint == 'impressions') { validateJsonApiPayload(impressionsSchema, rawPayload) + if (!rawPayload.range) { + throw new PayloadError('Range is required for impressions endpoint') + } + if (!isDataPayload(payload.data)) { + throw new PayloadError('Incorrect payload data type') + } await this.repo.storeImpressions( accountId, rawPayload.range.start, rawPayload.range.end, - payload.data as RawAnchorImpressionData + payload.data.data as RawAnchorImpressionData ) } else { throw new PayloadError( diff --git a/tsconfig.json b/tsconfig.json index 0ce95b84..4ae92290 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -91,5 +91,10 @@ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ - } + }, + "exclude": [ + "**/*.test.ts", + "**/*.test.js", + "node_modules" + ] } \ No newline at end of file