diff --git a/db_schema/migrations/19_spotify_graphql_raw.sql b/db_schema/migrations/19_spotify_graphql_raw.sql new file mode 100644 index 0000000..f24d6a3 --- /dev/null +++ b/db_schema/migrations/19_spotify_graphql_raw.sql @@ -0,0 +1,17 @@ +-- Migration 19: Create table for raw Spotify GraphQL data +-- Stores raw JSON payloads from the spotify_graphql pipeline +CREATE TABLE IF NOT EXISTS spotifyGraphQLRaw ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + account_id INTEGER NOT NULL, + show_uri VARCHAR(255) NOT NULL, + episode_uri VARCHAR(255) NULL, + endpoint VARCHAR(100) NOT NULL, + data JSON NOT NULL, + retrieved_at DATETIME NOT NULL, + PRIMARY KEY (id), + INDEX idx_graphql_raw_account_endpoint (account_id, endpoint), + INDEX idx_graphql_raw_account_show (account_id, show_uri), + INDEX idx_graphql_raw_account_episode (account_id, episode_uri) +); + +INSERT INTO migrations (migration_id, migration_name) VALUES (19, 'spotify graphql raw'); diff --git a/fixtures/spotifyGraphqlEpisodeAudienceSizeAllTime.json b/fixtures/spotifyGraphqlEpisodeAudienceSizeAllTime.json new file mode 100644 index 0000000..ed7949d --- /dev/null +++ b/fixtures/spotifyGraphqlEpisodeAudienceSizeAllTime.json @@ -0,0 +1,29 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "episodeAudienceSizeAllTime", + "episode": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "episodeByUri": { + "episodeAudienceSizeAllTime": { + "analyticsValue": { + "analyticsValue": { + "__typename": "SingleValueLong", + "value": 8 + } + }, + "endDate": "", + "startDate": "" + }, + "uri": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlEpisodeConsumptionAllTime.json b/fixtures/spotifyGraphqlEpisodeConsumptionAllTime.json new file mode 100644 index 0000000..ed5602f --- /dev/null +++ b/fixtures/spotifyGraphqlEpisodeConsumptionAllTime.json @@ -0,0 +1,33 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "episodeConsumptionAllTime", + "episode": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "episodeByUri": { + "episodeConsumptionAllTime": { + "analyticsValue": { + "analyticsValue": { + "__typename": "ConsumptionValue", + "backgroundMsPlayed": 0, + "foregroundConsumptionHours": 0.0, + "foregroundConsumptionPercent": 0.0, + "foregroundMsPlayed": 0, + "totalConsumptionHours": 3.62 + } + }, + "endDate": "", + "startDate": "" + }, + "uri": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlEpisodeMetadata.json b/fixtures/spotifyGraphqlEpisodeMetadata.json new file mode 100644 index 0000000..0214dbe --- /dev/null +++ b/fixtures/spotifyGraphqlEpisodeMetadata.json @@ -0,0 +1,35 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "episodeMetadata", + "episode": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "episodeByUri": { + "contentType": "EPISODE_CONTENT_TYPE_AUDIO", + "coverArt": { + "large": { + "url": "https://example.com/large.jpg" + }, + "small": { + "url": "https://example.com/small.jpg" + } + }, + "publishedOn": { + "seconds": 1773381600 + }, + "thumbnail": { + "images": [] + }, + "title": "Ep. 139: Example Episode", + "uri": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlEpisodePerformanceAllTime.json b/fixtures/spotifyGraphqlEpisodePerformanceAllTime.json new file mode 100644 index 0000000..a9d3941 --- /dev/null +++ b/fixtures/spotifyGraphqlEpisodePerformanceAllTime.json @@ -0,0 +1,57 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "episodePerformanceAllTime", + "episode": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "episodeByUri": { + "episodePerformanceTotalAllTime": { + "analyticsValue": { + "analyticsValue": { + "__typename": "PerformanceValue", + "medianCompletionSeconds": 72, + "percentiles": [ + { + "audiencePercentage": 50.0, + "completionPercentage": 25 + }, + { + "audiencePercentage": 25.0, + "completionPercentage": 50 + }, + { + "audiencePercentage": 13.0, + "completionPercentage": 75 + }, + { + "audiencePercentage": 13.0, + "completionPercentage": 100 + } + ], + "points": [ + { + "sampleCount": 8, + "second": 0 + }, + { + "sampleCount": 8, + "second": 1 + } + ] + } + }, + "endDate": "", + "startDate": "" + }, + "uri": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlEpisodePlaysDaily.json b/fixtures/spotifyGraphqlEpisodePlaysDaily.json new file mode 100644 index 0000000..d8ea242 --- /dev/null +++ b/fixtures/spotifyGraphqlEpisodePlaysDaily.json @@ -0,0 +1,37 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "episodePlaysDaily", + "episode": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "episodeByUri": { + "episodePlaysDaily": { + "analyticsValue": { + "analyticsValue": { + "__typename": "TimeSeriesValue", + "points": [ + { + "date": "2026-03-13", + "value": { + "__typename": "CountValueLong", + "value": 7 + } + } + ] + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "uri": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlEpisodeStreamsAndDownloads.json b/fixtures/spotifyGraphqlEpisodeStreamsAndDownloads.json new file mode 100644 index 0000000..2574701 --- /dev/null +++ b/fixtures/spotifyGraphqlEpisodeStreamsAndDownloads.json @@ -0,0 +1,46 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "episodeStreamsAndDownloads", + "episode": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "episodeByUri": { + "episodeStreamsAndDownloads": { + "analyticsValue": { + "analyticsValue": { + "__typename": "StreamsAndDownloadsValue", + "downloads": 0, + "streams": 0, + "downloadsTimeSeries": { + "__typename": "TimeSeriesValue", + "points": [ + { + "date": "2026-02-15", + "value": { + "__typename": "CountValueLong", + "value": 0 + } + } + ] + }, + "streamsTimeSeries": { + "__typename": "TimeSeriesValue", + "points": [] + } + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "uri": "spotify:episode:7tCwxfSBukfYDYvHD8QbK7" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowAudienceDiscovery.json b/fixtures/spotifyGraphqlShowAudienceDiscovery.json new file mode 100644 index 0000000..270be49 --- /dev/null +++ b/fixtures/spotifyGraphqlShowAudienceDiscovery.json @@ -0,0 +1,66 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showAudienceDiscovery" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "impressionsFunnel": { + "analyticsValue": { + "analyticsValue": { + "__typename": "FunnelValue", + "steps": [ + { + "conversionRate": 0.0, + "count": 0, + "displayName": "Reached", + "displayNameFull": "People you reached", + "stepName": "impressions" + } + ] + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "audienceSize": { + "analyticsValue": { + "analyticsValue": { + "__typename": "SingleValueLong", + "value": 0 + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "audienceFollowRate": { + "analyticsValue": { + "analyticsValue": { + "__typename": "PercentageValueFloat", + "value": 0.0 + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "consumptionHoursPerPerson": { + "analyticsValue": { + "analyticsValue": { + "__typename": "PercentageValueFloat", + "value": 0.0 + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowDemographicsStats.json b/fixtures/spotifyGraphqlShowDemographicsStats.json new file mode 100644 index 0000000..fad5214 --- /dev/null +++ b/fixtures/spotifyGraphqlShowDemographicsStats.json @@ -0,0 +1,31 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showDemographicsStats" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "showStreamsFaceted": { + "analyticsValue": { + "analyticsValue": { + "__typename": "FacetedAnalyticsValue", + "ageBreakdown": [], + "countryBreakdown": [], + "genderBreakdown": null, + "totalValue": 0 + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowGeoStats.json b/fixtures/spotifyGraphqlShowGeoStats.json new file mode 100644 index 0000000..04a579f --- /dev/null +++ b/fixtures/spotifyGraphqlShowGeoStats.json @@ -0,0 +1,26 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showGeoStats" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "showStreamsAndDownloadsByGeo": { + "analyticsValue": { + "analyticsValue": { + "__typename": "StreamsAndDownloadsByGeoValue", + "geos": [] + } + } + }, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowImpressionsSources.json b/fixtures/spotifyGraphqlShowImpressionsSources.json new file mode 100644 index 0000000..0e266ad --- /dev/null +++ b/fixtures/spotifyGraphqlShowImpressionsSources.json @@ -0,0 +1,19 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showImpressionsSources" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "impressionsSources": null, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowImpressionsTrend.json b/fixtures/spotifyGraphqlShowImpressionsTrend.json new file mode 100644 index 0000000..b8a2893 --- /dev/null +++ b/fixtures/spotifyGraphqlShowImpressionsTrend.json @@ -0,0 +1,37 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showImpressionsTrend" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "impressionsTotal": null, + "impressionsDaily": { + "analyticsValue": { + "analyticsValue": { + "__typename": "TimeSeriesValue", + "points": [ + { + "date": "2026-03-08", + "value": { + "__typename": "CountValueLong", + "value": 0 + } + } + ] + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowPlatformStats.json b/fixtures/spotifyGraphqlShowPlatformStats.json new file mode 100644 index 0000000..a4ebe4c --- /dev/null +++ b/fixtures/spotifyGraphqlShowPlatformStats.json @@ -0,0 +1,34 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showPlatformStats" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "showStreamsAndDownloadsByApp": { + "analyticsValue": { + "analyticsValue": { + "__typename": "StreamsAndDownloadsByAppValue", + "apps": [] + } + } + }, + "showStreamsAndDownloadsByDevice": { + "analyticsValue": { + "analyticsValue": { + "__typename": "StreamsAndDownloadsByDeviceValue", + "devices": [] + } + } + }, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowSpotifyStats.json b/fixtures/spotifyGraphqlShowSpotifyStats.json new file mode 100644 index 0000000..ba6ca26 --- /dev/null +++ b/fixtures/spotifyGraphqlShowSpotifyStats.json @@ -0,0 +1,92 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showSpotifyStats" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "playsDaily": { + "analyticsValue": { + "analyticsValue": { + "__typename": "TimeSeriesValue", + "points": [ + { + "date": "2026-03-10", + "value": { + "__typename": "CountValueLong", + "value": 0 + } + } + ] + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "consumptionHoursTotal": { + "analyticsValue": { + "analyticsValue": { + "__typename": "ConsumptionValue", + "totalConsumptionHours": 0.0 + } + } + }, + "consumptionHoursDaily": { + "analyticsValue": { + "analyticsValue": { + "__typename": "TimeSeriesValue", + "points": [ + { + "date": "2026-03-10", + "value": { + "__typename": "ConsumptionValue", + "backgroundMsPlayed": 0, + "foregroundMsPlayed": 0, + "totalConsumptionHours": 0.0 + } + } + ] + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "followerGrowthTotal": { + "analyticsValue": { + "analyticsValue": { + "__typename": "SingleValueLong", + "value": 0 + } + } + }, + "followersDaily": { + "analyticsValue": { + "analyticsValue": { + "__typename": "TimeSeriesValue", + "points": [] + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "followerGrowthDaily": { + "analyticsValue": { + "analyticsValue": { + "__typename": "TimeSeriesValue", + "points": [] + } + }, + "endDate": "2026-03-14", + "startDate": "2026-03-07" + }, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/fixtures/spotifyGraphqlShowTopEpisodes.json b/fixtures/spotifyGraphqlShowTopEpisodes.json new file mode 100644 index 0000000..f264a58 --- /dev/null +++ b/fixtures/spotifyGraphqlShowTopEpisodes.json @@ -0,0 +1,26 @@ +{ + "provider": "spotify", + "version": 1, + "retrieved": "2026-03-14T10:00:00.000Z", + "meta": { + "show": "spotify:show:0tJRC0UsObPCWLmmzmOkIs", + "endpoint": "showTopEpisodes" + }, + "range": { + "start": "2026-03-07", + "end": "2026-03-14" + }, + "data": { + "showByShowUri": { + "analytics": { + "analyticsValue": { + "analyticsValue": { + "__typename": "TopEpisodesValue", + "topEpisodes": [] + } + } + }, + "uri": "spotify:show:0tJRC0UsObPCWLmmzmOkIs" + } + } +} \ No newline at end of file diff --git a/src/api/connectors/SpotifyConnector.ts b/src/api/connectors/SpotifyConnector.ts index 9aa1b0c..b024296 100644 --- a/src/api/connectors/SpotifyConnector.ts +++ b/src/api/connectors/SpotifyConnector.ts @@ -14,6 +14,7 @@ import { SpotifyImpressionsDailyPayload, SpotifyImpressionsFacetedPayload, SpotifyImpressionsFunnelPayload, + SpotifyGraphQLPayload, } from '../../types/provider/spotify' import aggregateSchema from '../../schema/spotify/aggregate.json' import detailedStreamsSchema from '../../schema/spotify/detailedStreams.json' @@ -26,6 +27,20 @@ import impressionsTotalSchema from '../../schema/spotify/impressionsTotal.json' import impressionsDailySchema from '../../schema/spotify/impressionsDaily.json' import impressionsFacetedSchema from '../../schema/spotify/impressionsFaceted.json' import impressionsFunnelSchema from '../../schema/spotify/impressionsFunnel.json' +import showSpotifyStatsSchema from '../../schema/spotify/showSpotifyStats.json' +import showDemographicsStatsSchema from '../../schema/spotify/showDemographicsStats.json' +import showAudienceDiscoverySchema from '../../schema/spotify/showAudienceDiscovery.json' +import showGeoStatsSchema from '../../schema/spotify/showGeoStats.json' +import showPlatformStatsSchema from '../../schema/spotify/showPlatformStats.json' +import showImpressionsTrendSchema from '../../schema/spotify/showImpressionsTrend.json' +import showImpressionsSourcesSchema from '../../schema/spotify/showImpressionsSources.json' +import showTopEpisodesSchema from '../../schema/spotify/showTopEpisodes.json' +import episodeMetadataGraphQLSchema from '../../schema/spotify/episodeMetadataGraphQL.json' +import episodePerformanceAllTimeSchema from '../../schema/spotify/episodePerformanceAllTime.json' +import episodeStreamsAndDownloadsSchema from '../../schema/spotify/episodeStreamsAndDownloads.json' +import episodePlaysGraphQLDailySchema from '../../schema/spotify/episodePlaysDaily.json' +import episodeConsumptionAllTimeSchema from '../../schema/spotify/episodeConsumptionAllTime.json' +import episodeAudienceSizeAllTimeSchema from '../../schema/spotify/episodeAudienceSizeAllTime.json' import { validateJsonApiPayload } from '../JsonPayloadValidator' import { SpotifyRepository } from '../../db/SpotifyRepository' @@ -67,13 +82,28 @@ class SpotifyConnector implements ConnectorHandler { ) } } else if (payload.meta.endpoint === 'episodeMetadata') { - // metadata of single episode - validateJsonApiPayload(episodeMetadataSchema, payload.data) - - return await this.repo.storeEpisodeMetadata( - accountId, - payload.data as SpotifyEpisodeMetadataPayload - ) + // Handle both the old Spotify API shape (flat fields) and the new + // GraphQL pipeline shape ({ episodeByUri: { ... } }). + const episodeData = payload.data as any + if (episodeData.episodeByUri !== undefined) { + // New GraphQL pipeline shape + validateJsonApiPayload(episodeMetadataGraphQLSchema, episodeData) + return await this.repo.storeGraphQLRaw( + accountId, + payload.meta.show as string, + payload.meta.episode, + 'episodeMetadata', + payload.data, + payload.retrieved + ) + } else { + // Old Spotify API shape + validateJsonApiPayload(episodeMetadataSchema, payload.data) + return await this.repo.storeEpisodeMetadata( + accountId, + payload.data as SpotifyEpisodeMetadataPayload + ) + } } else if (payload.meta.endpoint === 'followers') { // follower count per day validateJsonApiPayload(followerSchema, payload.data) @@ -174,6 +204,45 @@ class SpotifyConnector implements ConnectorHandler { accountId, payload.data as SpotifyImpressionsFunnelPayload ) + } else if (payload.meta.endpoint === 'showSpotifyStats') { + validateJsonApiPayload(showSpotifyStatsSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showSpotifyStats', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'showDemographicsStats') { + validateJsonApiPayload(showDemographicsStatsSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showDemographicsStats', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'showAudienceDiscovery') { + validateJsonApiPayload(showAudienceDiscoverySchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showAudienceDiscovery', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'showGeoStats') { + validateJsonApiPayload(showGeoStatsSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showGeoStats', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'showPlatformStats') { + validateJsonApiPayload(showPlatformStatsSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showPlatformStats', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'showImpressionsTrend') { + validateJsonApiPayload(showImpressionsTrendSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showImpressionsTrend', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'showImpressionsSources') { + validateJsonApiPayload(showImpressionsSourcesSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showImpressionsSources', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'showTopEpisodes') { + validateJsonApiPayload(showTopEpisodesSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'showTopEpisodes', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'episodePerformanceAllTime') { + validateJsonApiPayload(episodePerformanceAllTimeSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'episodePerformanceAllTime', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'episodeStreamsAndDownloads') { + validateJsonApiPayload(episodeStreamsAndDownloadsSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'episodeStreamsAndDownloads', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'episodePlaysDaily') { + validateJsonApiPayload(episodePlaysGraphQLDailySchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'episodePlaysDaily', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'episodeConsumptionAllTime') { + validateJsonApiPayload(episodeConsumptionAllTimeSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'episodeConsumptionAllTime', payload.data, payload.retrieved) + } else if (payload.meta.endpoint === 'episodeAudienceSizeAllTime') { + validateJsonApiPayload(episodeAudienceSizeAllTimeSchema, payload.data) + return await this.repo.storeGraphQLRaw(accountId, payload.meta.show as string, payload.meta.episode, 'episodeAudienceSizeAllTime', payload.data, payload.retrieved) } else { throw new PayloadError( `Unknown endpoint in meta: ${payload.meta.endpoint}` diff --git a/src/db/DBInitializer.ts b/src/db/DBInitializer.ts index be92b6c..243f6e1 100644 --- a/src/db/DBInitializer.ts +++ b/src/db/DBInitializer.ts @@ -146,7 +146,7 @@ class DBInitializer { migrationId++ ) { console.log(`Running migration number ${migrationId} ...`) - this.runMigration(migrationId) + await this.runMigration(migrationId) console.log(`Migration ${migrationId} done`) } console.log('All migration work finished') diff --git a/src/db/SpotifyRepository.ts b/src/db/SpotifyRepository.ts index 8a17692..b193ecc 100644 --- a/src/db/SpotifyRepository.ts +++ b/src/db/SpotifyRepository.ts @@ -494,6 +494,33 @@ class SpotifyRepository { return Promise.all(queryPromises) } + + async storeGraphQLRaw( + accountId: number, + showUri: string, + episodeUri: string | null | undefined, + endpoint: string, + data: object, + retrievedAt: string + ): Promise { + const insertStmt = `INSERT INTO spotifyGraphQLRaw ( + account_id, + show_uri, + episode_uri, + endpoint, + data, + retrieved_at + ) VALUES (?,?,?,?,?,?)` + + return await this.pool.query(insertStmt, [ + accountId, + showUri, + episodeUri ?? null, + endpoint, + JSON.stringify(data), + retrievedAt, + ]) + } } export { SpotifyRepository } diff --git a/src/schema/spotify/episodeAudienceSizeAllTime.json b/src/schema/spotify/episodeAudienceSizeAllTime.json new file mode 100644 index 0000000..1d78253 --- /dev/null +++ b/src/schema/spotify/episodeAudienceSizeAllTime.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/episodeAudienceSizeAllTime.schema.json", + "title": "Spotify episode audience size all time (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/episodeConsumptionAllTime.json b/src/schema/spotify/episodeConsumptionAllTime.json new file mode 100644 index 0000000..c6d8e7f --- /dev/null +++ b/src/schema/spotify/episodeConsumptionAllTime.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/episodeConsumptionAllTime.schema.json", + "title": "Spotify episode consumption all time (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/episodeMetadataGraphQL.json b/src/schema/spotify/episodeMetadataGraphQL.json new file mode 100644 index 0000000..8e30724 --- /dev/null +++ b/src/schema/spotify/episodeMetadataGraphQL.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/episodeMetadataGraphQL.schema.json", + "title": "Spotify episode metadata from GraphQL pipeline", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/episodePerformanceAllTime.json b/src/schema/spotify/episodePerformanceAllTime.json new file mode 100644 index 0000000..cc7306c --- /dev/null +++ b/src/schema/spotify/episodePerformanceAllTime.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/episodePerformanceAllTime.schema.json", + "title": "Spotify episode performance all time (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/episodePlaysDaily.json b/src/schema/spotify/episodePlaysDaily.json new file mode 100644 index 0000000..1c40239 --- /dev/null +++ b/src/schema/spotify/episodePlaysDaily.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/episodePlaysDaily.schema.json", + "title": "Spotify episode plays daily (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/episodeStreamsAndDownloads.json b/src/schema/spotify/episodeStreamsAndDownloads.json new file mode 100644 index 0000000..dd1b528 --- /dev/null +++ b/src/schema/spotify/episodeStreamsAndDownloads.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/episodeStreamsAndDownloads.schema.json", + "title": "Spotify episode streams and downloads (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showAudienceDiscovery.json b/src/schema/spotify/showAudienceDiscovery.json new file mode 100644 index 0000000..88acc81 --- /dev/null +++ b/src/schema/spotify/showAudienceDiscovery.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showAudienceDiscovery.schema.json", + "title": "Spotify show audience discovery (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showDemographicsStats.json b/src/schema/spotify/showDemographicsStats.json new file mode 100644 index 0000000..a3ae16f --- /dev/null +++ b/src/schema/spotify/showDemographicsStats.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showDemographicsStats.schema.json", + "title": "Spotify show demographics (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showGeoStats.json b/src/schema/spotify/showGeoStats.json new file mode 100644 index 0000000..75f74ab --- /dev/null +++ b/src/schema/spotify/showGeoStats.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showGeoStats.schema.json", + "title": "Spotify show geo stats (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showImpressionsSources.json b/src/schema/spotify/showImpressionsSources.json new file mode 100644 index 0000000..6550b85 --- /dev/null +++ b/src/schema/spotify/showImpressionsSources.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showImpressionsSources.schema.json", + "title": "Spotify show impressions sources (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showImpressionsTrend.json b/src/schema/spotify/showImpressionsTrend.json new file mode 100644 index 0000000..ff10ca7 --- /dev/null +++ b/src/schema/spotify/showImpressionsTrend.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showImpressionsTrend.schema.json", + "title": "Spotify show impressions trend (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showPlatformStats.json b/src/schema/spotify/showPlatformStats.json new file mode 100644 index 0000000..efaa913 --- /dev/null +++ b/src/schema/spotify/showPlatformStats.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showPlatformStats.schema.json", + "title": "Spotify show platform stats (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showSpotifyStats.json b/src/schema/spotify/showSpotifyStats.json new file mode 100644 index 0000000..880e255 --- /dev/null +++ b/src/schema/spotify/showSpotifyStats.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showSpotifyStats.schema.json", + "title": "Spotify show daily plays, consumption, followers (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/schema/spotify/showTopEpisodes.json b/src/schema/spotify/showTopEpisodes.json new file mode 100644 index 0000000..106984d --- /dev/null +++ b/src/schema/spotify/showTopEpisodes.json @@ -0,0 +1,7 @@ +{ + "$id": "http://openpodcast.dev/schema/spotify/showTopEpisodes.schema.json", + "title": "Spotify show top episodes (GraphQL)", + "type": "object", + "additionalProperties": true, + "properties": {} +} \ No newline at end of file diff --git a/src/types/connector.ts b/src/types/connector.ts index 604e2ee..212ca1e 100644 --- a/src/types/connector.ts +++ b/src/types/connector.ts @@ -10,4 +10,6 @@ export interface ConnectorPayload { } data: object provider: string + retrieved: string + version?: number } diff --git a/src/types/provider/spotify.ts b/src/types/provider/spotify.ts index a3625d7..7ec43f8 100644 --- a/src/types/provider/spotify.ts +++ b/src/types/provider/spotify.ts @@ -146,3 +146,21 @@ export interface SpotifyImpressionsFunnelPayload { conversionPercent?: number }[] } + +// --------------------------------------------------------------------------- +// Spotify GraphQL pipeline types (provider: spotify, spotify_graphql pipeline) +// --------------------------------------------------------------------------- + +// Generic wrapper for all GraphQL payloads - the actual shape is deeply nested +// GraphQL-specific, so we use 'any' for the inner structure. + +export interface SpotifyGraphQLShowPayload { + showByShowUri: any +} + +export interface SpotifyGraphQLEpisodePayload { + episodeByUri: any +} + +// Union type that covers both shapes +export type SpotifyGraphQLPayload = SpotifyGraphQLShowPayload | SpotifyGraphQLEpisodePayload