From db61231622abf8033ffc3a32614f7ce1df1a0767 Mon Sep 17 00:00:00 2001 From: Anlor Date: Sat, 23 May 2026 08:03:00 +0000 Subject: [PATCH 1/8] fix: update delta query parameters to exclude audio field --- src/graph/graph.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/graph/graph.ts b/src/graph/graph.ts index e163eaa..77f1224 100644 --- a/src/graph/graph.ts +++ b/src/graph/graph.ts @@ -149,7 +149,8 @@ export const getDelta = async ( const queryParams = { $top: '2147483647', - $select: 'id,name,parentReference,folder,cTag,deleted,size,lastModifiedDateTime,audio' + // 目前 /delta 接口不会返回 audio 字段 + $select: 'id,name,parentReference,folder,cTag,deleted,size,lastModifiedDateTime' } const param = new URLSearchParams(queryParams) From 267ac47df6c58f7ba59cb7bbbf1f3dbab74f3025 Mon Sep 17 00:00:00 2001 From: Anlor Date: Sat, 23 May 2026 14:06:12 +0000 Subject: [PATCH 2/8] feat: add priority handling for Graph requests and implement rate limiting --- src/graph/graph.ts | 19 ++- src/graph/rateLimiter.ts | 256 ++++++++++++++++++++++++++++++++++++ src/hooks/graph/useGraph.ts | 29 ++-- 3 files changed, 284 insertions(+), 20 deletions(-) create mode 100644 src/graph/rateLimiter.ts diff --git a/src/graph/graph.ts b/src/graph/graph.ts index 77f1224..c3828c0 100644 --- a/src/graph/graph.ts +++ b/src/graph/graph.ts @@ -1,11 +1,13 @@ import { DeltaResponse, FileResponse, RemoteItem } from '@/types/file' import { graphConfig } from './authConfig' +import { graphFetch, GraphRequestPriority } from './rateLimiter' export async function getFiles( accessToken: string, id: string, path?: string[], nextLink?: string, + priority?: GraphRequestPriority, ): Promise { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -30,7 +32,7 @@ export async function getFiles( : `${graphConfig.graphMeEndpoint}/me/drive/root:/${encodeURIComponent(path.join('/'))}:/children?${params.toString()}` : `${graphConfig.graphMeEndpoint}/me/drive/items/${id}/children?${params.toString()}` - return fetch(nextLink || url, options) + return graphFetch(nextLink || url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } @@ -40,6 +42,7 @@ export async function getFile( id: string, path?: string[], signal?: AbortSignal, + priority?: GraphRequestPriority, ): Promise { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -62,13 +65,14 @@ export async function getFile( ? `${graphConfig.graphMeEndpoint}/me/drive/root:/${encodeURIComponent(path.join('/'))}?${params.toString()}` : `${graphConfig.graphMeEndpoint}/me/drive/items/${id}?${params.toString()}` - return fetch(url, options) + return graphFetch(url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } export const getAppRootFiles = async ( accessToken: string, + priority?: GraphRequestPriority, ) => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -82,7 +86,7 @@ export const getAppRootFiles = async ( const url = `${graphConfig.graphMeEndpoint}/me/drive/special/approot/children` - return fetch(url, options) + return graphFetch(url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } @@ -91,6 +95,7 @@ export const uploadAppRootJson = async ( accessToken: string, fileName: string, fileContent: BodyInit, + priority?: GraphRequestPriority, ) => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -106,7 +111,7 @@ export const uploadAppRootJson = async ( const url = `${graphConfig.graphMeEndpoint}/me/drive/special/approot:/${fileName}:/content` - return fetch(url, options) + return graphFetch(url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } @@ -114,6 +119,7 @@ export const uploadAppRootJson = async ( export const search = async ( accessToken: string, searchQuery: string, + priority?: GraphRequestPriority, ): Promise => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -127,7 +133,7 @@ export const search = async ( const url = `${graphConfig.graphMeEndpoint}/me/drive/root/search(q='${searchQuery}')` - return fetch(url, options) + return graphFetch(url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } @@ -136,6 +142,7 @@ export const getDelta = async ( accessToken: string, id?: string, deltaLink?: string, + priority?: GraphRequestPriority, ): Promise => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -159,7 +166,7 @@ export const getDelta = async ( ? `${graphConfig.graphMeEndpoint}/me/drive/items/${id}/delta?${param.toString()}` : `${graphConfig.graphMeEndpoint}/me/drive/root/delta?${param.toString()}` - return fetch(deltaLink || url, options) + return graphFetch(deltaLink || url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } \ No newline at end of file diff --git a/src/graph/rateLimiter.ts b/src/graph/rateLimiter.ts new file mode 100644 index 0000000..8355a84 --- /dev/null +++ b/src/graph/rateLimiter.ts @@ -0,0 +1,256 @@ +export type GraphRequestPriority = 'high' | 'low' + +export interface GraphFetchOptions { + priority?: GraphRequestPriority + maxRetries?: number +} + +type ResolvedGraphFetchOptions = Required + +interface GraphRateLimitContext { + input: RequestInfo | URL + method?: string + priority: GraphRequestPriority + retry: number + maxRetries: number +} + +const DEFAULT_MAX_RETRIES = 2 +const THROTTLED_STATUS_CODES = new Set([429, 503]) + +// Microsoft Graph recommends exponential backoff when Retry-After isn't returned: +// https://learn.microsoft.com/en-us/graph/throttling +// The Microsoft Graph SDK retry delay defaults to 3 seconds and allows up to 180 seconds: +// https://learn.microsoft.com/en-us/dotnet/api/microsoft.graph.retryhandleroption.delay +const FALLBACK_RETRY_BASE_DELAY_MS = 3_000 +const FALLBACK_RETRY_MAX_DELAY_MS = 180_000 + +let globalBackoffUntil = 0 +let lowPriorityBackoffUntil = 0 +let lowPriorityQueue: Promise = Promise.resolve() + +export const parseRetryAfterMs = (retryAfter: string | null): number | undefined => { + if (!retryAfter) return undefined + + const seconds = Number(retryAfter) + if (Number.isFinite(seconds)) { + return Math.max(0, seconds * 1000) + } + + const date = Date.parse(retryAfter) + if (Number.isNaN(date)) return undefined + + return Math.max(0, date - Date.now()) +} + +export const parseRateLimitResetMs = (rateLimitReset: string | null): number | undefined => { + if (!rateLimitReset) return undefined + + const seconds = Number(rateLimitReset) + if (Number.isFinite(seconds)) { + return Math.max(0, seconds * 1000) + } + + const date = Date.parse(rateLimitReset) + if (Number.isNaN(date)) return undefined + + return Math.max(0, date - Date.now()) +} + +export const getGraphRetryDelayMs = (headers: Headers): number | undefined => { + const retryAfterMs = parseRetryAfterMs(headers.get('Retry-After')) + const rateLimitResetMs = parseRateLimitResetMs(headers.get('RateLimit-Reset')) + const delays = [retryAfterMs, rateLimitResetMs].filter((delay): delay is number => delay !== undefined) + + if (delays.length === 0) return undefined + + return Math.max(...delays) +} + +const getFallbackRetryDelayMs = (retry: number) => + Math.min(FALLBACK_RETRY_BASE_DELAY_MS * (2 ** retry), FALLBACK_RETRY_MAX_DELAY_MS) + +const getRequestUrl = (input: RequestInfo | URL) => { + if (typeof input === 'string') return input + if (input instanceof URL) return input.toString() + + return input.url +} + +const logGraphRateLimit = ( + message: string, + context: GraphRateLimitContext, + headers: Headers, + status: number, + delayMs: number, +) => { + console.warn(`[Graph] ${message}`, { + status, + method: context.method ?? 'GET', + url: getRequestUrl(context.input), + priority: context.priority, + retry: context.retry, + maxRetries: context.maxRetries, + delayMs, + retryAfter: headers.get('Retry-After'), + rateLimitReset: headers.get('RateLimit-Reset'), + rateLimitRemaining: headers.get('RateLimit-Remaining'), + rateLimitLimit: headers.get('RateLimit-Limit'), + throttleLimitPercentage: headers.get('x-ms-throttle-limit-percentage'), + }) +} + +const sleep = (ms: number, signal?: AbortSignal): Promise => { + if (ms <= 0) return Promise.resolve() + + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason ?? new DOMException('Aborted', 'AbortError')) + return + } + + const timeout = window.setTimeout(() => { + signal?.removeEventListener('abort', handleAbort) + resolve() + }, ms) + + const handleAbort = () => { + window.clearTimeout(timeout) + reject(signal?.reason ?? new DOMException('Aborted', 'AbortError')) + } + + signal?.addEventListener('abort', handleAbort, { once: true }) + }) +} + +const waitForBackoff = async (priority: GraphRequestPriority, signal?: AbortSignal) => { + const now = Date.now() + const backoffUntil = priority === 'low' + ? Math.max(globalBackoffUntil, lowPriorityBackoffUntil) + : globalBackoffUntil + + await sleep(backoffUntil - now, signal) +} + +const updateBackoffFromHeaders = (headers: Headers, status: number, context: GraphRateLimitContext) => { + const now = Date.now() + const retryDelayMs = getGraphRetryDelayMs(headers) + const rateLimitRemainingHeader = headers.get('RateLimit-Remaining') + const rateLimitLimitHeader = headers.get('RateLimit-Limit') + const rateLimitRemaining = rateLimitRemainingHeader === null ? undefined : Number(rateLimitRemainingHeader) + const rateLimitLimit = rateLimitLimitHeader === null ? undefined : Number(rateLimitLimitHeader) + const isGlobalBackoffTriggered = THROTTLED_STATUS_CODES.has(status) || rateLimitRemaining === 0 + let hasLoggedLowPriorityBackoff = false + + if (isGlobalBackoffTriggered) { + const delayMs = retryDelayMs ?? getFallbackRetryDelayMs(context.retry) + globalBackoffUntil = Math.max(globalBackoffUntil, now + delayMs) + logGraphRateLimit('Rate limit triggered; backing off Graph requests.', context, headers, status, delayMs) + } + + // Microsoft Graph files/lists resources (drive, driveItem, etc.) use SharePoint limits: + // https://learn.microsoft.com/en-us/graph/throttling-limits#files-and-lists-service-limits + // RateLimit headers are emitted near quota pressure and include RateLimit-Reset; + // https://learn.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online#ratelimit-headers---preview + if ( + rateLimitRemaining !== undefined + && rateLimitLimit !== undefined + && Number.isFinite(rateLimitRemaining) + && Number.isFinite(rateLimitLimit) + && rateLimitLimit > 0 + && rateLimitRemaining / rateLimitLimit < 0.2 + ) { + const delayMs = retryDelayMs ?? FALLBACK_RETRY_BASE_DELAY_MS + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) + if (!isGlobalBackoffTriggered) { + logGraphRateLimit('Rate limit quota is low; backing off low-priority Graph requests.', context, headers, status, delayMs) + hasLoggedLowPriorityBackoff = true + } + } + + const throttleLimitPercentage = Number(headers.get('x-ms-throttle-limit-percentage')) + if (Number.isFinite(throttleLimitPercentage) && throttleLimitPercentage >= 0.8) { + const delayMs = retryDelayMs ?? FALLBACK_RETRY_BASE_DELAY_MS + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) + if (!isGlobalBackoffTriggered && !hasLoggedLowPriorityBackoff) { + logGraphRateLimit('Throttle limit is high; backing off low-priority Graph requests.', context, headers, status, delayMs) + } + } +} + +const runGraphFetch = async ( + input: RequestInfo | URL, + init: RequestInit | undefined, + options: ResolvedGraphFetchOptions, +): Promise => { + const signal = init?.signal ?? undefined + + for (let retry = 0; retry <= options.maxRetries; retry += 1) { + await waitForBackoff(options.priority, signal) + + const headers = new Headers(init?.headers) + + // Priority header docs: + // https://learn.microsoft.com/en-us/graph/throttling-limits + headers.set('x-ms-throttle-priority', options.priority === 'low' ? 'Low' : 'High') + + const response = await fetch(input, { + ...init, + headers, + }) + + updateBackoffFromHeaders(response.headers, response.status, { + input, + method: init?.method, + priority: options.priority, + retry, + maxRetries: options.maxRetries, + }) + + if (!THROTTLED_STATUS_CODES.has(response.status) || retry >= options.maxRetries) { + return response + } + } + + throw new Error('Unexpected Graph fetch retry state.') +} + +const enqueueLowPriorityFetch = ( + input: RequestInfo | URL, + init: RequestInit | undefined, + options: ResolvedGraphFetchOptions, +): Promise => { + const previousQueue = lowPriorityQueue.catch(() => undefined) + let releaseQueue: () => void = () => undefined + + lowPriorityQueue = previousQueue.then( + () => new Promise((resolve) => { + releaseQueue = resolve + }) + ) + + return previousQueue.then(async () => { + try { + return await runGraphFetch(input, init, options) + } finally { + releaseQueue() + } + }) +} + +export const graphFetch = ( + input: RequestInfo | URL, + init?: RequestInit, + { + priority = 'high', + maxRetries = DEFAULT_MAX_RETRIES, + }: GraphFetchOptions = {}, +): Promise => { + const options = { priority, maxRetries } + + if (priority === 'low') { + return enqueueLowPriorityFetch(input, init, options) + } + + return runGraphFetch(input, init, options) +} diff --git a/src/hooks/graph/useGraph.ts b/src/hooks/graph/useGraph.ts index b46b275..0af4a85 100644 --- a/src/hooks/graph/useGraph.ts +++ b/src/hooks/graph/useGraph.ts @@ -1,6 +1,7 @@ import { getAppRootFiles, getFile, getFiles, search, uploadAppRootJson, getDelta } from '@/graph/graph' import { loginRequest } from '@/graph/authConfig' import { AccountInfo, IPublicClientApplication } from '@azure/msal-browser' +import { GraphRequestPriority } from '@/graph/rateLimiter' const useGraph = ( instance: IPublicClientApplication, @@ -20,54 +21,54 @@ const useGraph = ( return tokenResponse.accessToken } - const getFilesData = async (id: string, path?: string[]) => { + const getFilesData = async (id: string, path?: string[], priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - let response = await getFiles(accessToken, id, path,) + let response = await getFiles(accessToken, id, path, undefined, priority) const remoteItems = [...response.value] while (response['@odata.nextLink']) { - response = await getFiles(accessToken, id, path, response['@odata.nextLink']) + response = await getFiles(accessToken, id, path, response['@odata.nextLink'], priority) remoteItems.push(...response.value) } return { value: remoteItems } } - const getFileData = async (id: string, path?: string[], signal?: AbortSignal) => { + const getFileData = async (id: string, path?: string[], signal?: AbortSignal, priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - const response = await getFile(accessToken, id, path, signal) + const response = await getFile(accessToken, id, path, signal, priority) return response } - const getAppRootFilesData = async () => { + const getAppRootFilesData = async (priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - const response = await getAppRootFiles(accessToken) + const response = await getAppRootFiles(accessToken, priority) return response } - const uploadAppRootJsonData = async (fileName: string, fileContent: BodyInit) => { + const uploadAppRootJsonData = async (fileName: string, fileContent: BodyInit, priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - const response = await uploadAppRootJson(accessToken, fileName, fileContent) + const response = await uploadAppRootJson(accessToken, fileName, fileContent, priority) return response } - const getSearchData = async (searchQuery: string) => { + const getSearchData = async (searchQuery: string, priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - const response = await search(accessToken, searchQuery) + const response = await search(accessToken, searchQuery, priority) return response } - const getDeltaData = async (id: string, deltaLink?: string) => { + const getDeltaData = async (id: string, deltaLink?: string, priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - let response = await getDelta(accessToken, id, deltaLink) + let response = await getDelta(accessToken, id, deltaLink, priority) const remoteItems = [...response.value] while (response['@odata.nextLink']) { - response = await getDelta(accessToken, id, response['@odata.nextLink']) + response = await getDelta(accessToken, id, response['@odata.nextLink'], priority) remoteItems.push(...response.value) } From 2d7f7f55cc71f8a53222319e3ae37570a08ebf5e Mon Sep 17 00:00:00 2001 From: Anlor Date: Sat, 23 May 2026 15:39:26 +0000 Subject: [PATCH 3/8] feat: add p-limit for low priority request handling in graphFetch --- package-lock.json | 50 +++++++++++++++++++++++++++++++--------- package.json | 3 ++- src/graph/rateLimiter.ts | 24 ++++--------------- 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/package-lock.json b/package-lock.json index ae99a8f..7115970 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "franc-min": "^6.2.0", "idb-keyval": "^6.2.1", "music-metadata": "^11.9.0", + "p-limit": "^7.3.0", "pinyin-pro": "^3.27.0", "process": "^0.11.10", "react": "^18.3.1", @@ -9998,16 +9999,15 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "7.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", "license": "MIT", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.2.1" }, "engines": { - "node": ">=10" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -10029,6 +10029,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmmirror.com/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-retry": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", @@ -14053,13 +14082,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.2.2", + "resolved": "https://registry.npmmirror.com/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index 6002f0a..4b1ef56 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "franc-min": "^6.2.0", "idb-keyval": "^6.2.1", "music-metadata": "^11.9.0", + "p-limit": "^7.3.0", "pinyin-pro": "^3.27.0", "process": "^0.11.10", "react": "^18.3.1", @@ -88,4 +89,4 @@ "webpack-merge": "^6.0.1", "workbox-webpack-plugin": "^7.3.0" } -} \ No newline at end of file +} diff --git a/src/graph/rateLimiter.ts b/src/graph/rateLimiter.ts index 8355a84..85d8079 100644 --- a/src/graph/rateLimiter.ts +++ b/src/graph/rateLimiter.ts @@ -1,3 +1,5 @@ +import pLimit from 'p-limit' + export type GraphRequestPriority = 'high' | 'low' export interface GraphFetchOptions { @@ -24,10 +26,11 @@ const THROTTLED_STATUS_CODES = new Set([429, 503]) // https://learn.microsoft.com/en-us/dotnet/api/microsoft.graph.retryhandleroption.delay const FALLBACK_RETRY_BASE_DELAY_MS = 3_000 const FALLBACK_RETRY_MAX_DELAY_MS = 180_000 +const LOW_PRIORITY_CONCURRENCY = 2 let globalBackoffUntil = 0 let lowPriorityBackoffUntil = 0 -let lowPriorityQueue: Promise = Promise.resolve() +const lowPriorityLimit = pLimit(LOW_PRIORITY_CONCURRENCY) export const parseRetryAfterMs = (retryAfter: string | null): number | undefined => { if (!retryAfter) return undefined @@ -219,24 +222,7 @@ const enqueueLowPriorityFetch = ( input: RequestInfo | URL, init: RequestInit | undefined, options: ResolvedGraphFetchOptions, -): Promise => { - const previousQueue = lowPriorityQueue.catch(() => undefined) - let releaseQueue: () => void = () => undefined - - lowPriorityQueue = previousQueue.then( - () => new Promise((resolve) => { - releaseQueue = resolve - }) - ) - - return previousQueue.then(async () => { - try { - return await runGraphFetch(input, init, options) - } finally { - releaseQueue() - } - }) -} +): Promise => lowPriorityLimit(() => runGraphFetch(input, init, options)) export const graphFetch = ( input: RequestInfo | URL, From c44a02ea3b2cbbde6f76290d84ac6cb58681810d Mon Sep 17 00:00:00 2001 From: Anlor Date: Sun, 12 Jul 2026 16:59:37 +0000 Subject: [PATCH 4/8] feat: add metadata auto syncing for musics in library --- package.json | 1 + src/App.tsx | 4 +- src/graph/graph.ts | 55 +++-- src/graph/rateLimiter.ts | 157 ++++++++---- src/hooks/graph/useGraph.ts | 11 +- src/hooks/graph/useMetadataSync.ts | 109 +++++++++ src/hooks/player/useMetaData.ts | 11 +- src/locales/en/messages.po | 16 +- src/locales/zh-CN/messages.po | 16 +- src/pages/Library/Library.tsx | 10 +- src/store/useMetadataSyncStore.ts | 20 ++ src/types/MetaData.ts | 3 +- src/types/file.ts | 24 +- src/utils/getNetMetaData.ts | 7 +- src/utils/graphAudioMetadata.ts | 47 ++++ src/utils/rangeMetadata.test.ts | 132 ++++++++++ src/utils/rangeMetadata.ts | 371 +++++++++++++++++++++++++++++ src/utils/rangeReader.ts | 117 +++++++++ src/utils/remote.ts | 8 +- tsconfig.json | 3 +- 20 files changed, 1017 insertions(+), 105 deletions(-) create mode 100644 src/hooks/graph/useMetadataSync.ts create mode 100644 src/store/useMetadataSyncStore.ts create mode 100644 src/utils/graphAudioMetadata.ts create mode 100644 src/utils/rangeMetadata.test.ts create mode 100644 src/utils/rangeMetadata.ts create mode 100644 src/utils/rangeReader.ts diff --git a/package.json b/package.json index 4b1ef56..cc13650 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "dev": "npm run generate:dev && webpack serve", "lint": "npm run generate && eslint src --report-unused-disable-directives --max-warnings 0", "lint:dev": "npm run generate:dev && eslint src --report-unused-disable-directives --max-warnings 0", + "test": "node --experimental-strip-types --test src/utils/rangeMetadata.test.ts", "lingui": "lingui extract && lingui compile --typescript", "generate": "node generate-version-info.mjs && npm run lingui", "generate:dev": "node generate-version-info.mjs --dev && npm run lingui" diff --git a/src/App.tsx b/src/App.tsx index 2e988f5..8ab9478 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -20,6 +20,7 @@ import useFileNodeSyncStore from './store/useFileNodeSyncStore' import { useLiveQuery } from 'dexie-react-hooks' import useDb from './hooks/useDb' import useTitle from './hooks/ui/useTitle' +import useMetadataSync from './hooks/graph/useMetadataSync' const App = () => { useEnvironment() @@ -32,6 +33,7 @@ const App = () => { const { account } = useUser() useSync() useFileNodeSync() + useMetadataSync() const db = useDb(account) const libraryRootId = useLiveQuery(async () => (await db?.settings.get('settings'))?.libraryRootId, [db]) @@ -141,4 +143,4 @@ const App = () => { ) } -export default App \ No newline at end of file +export default App diff --git a/src/graph/graph.ts b/src/graph/graph.ts index c3828c0..6df2389 100644 --- a/src/graph/graph.ts +++ b/src/graph/graph.ts @@ -37,37 +37,44 @@ export async function getFiles( .catch(error => console.log(error)) } -export async function getFile( +const parseGraphResponse = async (response: Response): Promise => { + if (!response.ok) { + throw new Error(`Graph request failed with status ${response.status}.`) + } + + return response.json() as Promise +} + +const requestDriveItem = async ( accessToken: string, id: string, path?: string[], signal?: AbortSignal, priority?: GraphRequestPriority, -): Promise { - const headers = new Headers() - const bearer = `Bearer ${accessToken}` - - headers.append('Authorization', bearer) - - const options = { - method: 'GET', - headers: headers, - signal: signal, - } - - const queryParams = { - $expand: 'thumbnails' - } - - const params = new URLSearchParams(queryParams) + includeThumbnails = false, +): Promise => { + const headers = new Headers({ Authorization: `Bearer ${accessToken}` }) + const query = includeThumbnails + ? `?${new URLSearchParams({ $expand: 'thumbnails' }).toString()}` + : '' const url = path - ? `${graphConfig.graphMeEndpoint}/me/drive/root:/${encodeURIComponent(path.join('/'))}?${params.toString()}` - : `${graphConfig.graphMeEndpoint}/me/drive/items/${id}?${params.toString()}` + ? `${graphConfig.graphMeEndpoint}/me/drive/root:/${encodeURIComponent(path.join('/'))}${query}` + : `${graphConfig.graphMeEndpoint}/me/drive/items/${id}${query}` + const response = await graphFetch(url, { method: 'GET', headers, signal }, { priority }) - return graphFetch(url, options, { priority }) - .then(response => response.json()) - .catch(error => console.log(error)) + return parseGraphResponse(response) +} + +export async function getFile( + accessToken: string, + id: string, + path?: string[], + signal?: AbortSignal, + priority?: GraphRequestPriority, + includeThumbnails = true, +): Promise { + return requestDriveItem(accessToken, id, path, signal, priority, includeThumbnails) } export const getAppRootFiles = async ( @@ -169,4 +176,4 @@ export const getDelta = async ( return graphFetch(deltaLink || url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) -} \ No newline at end of file +} diff --git a/src/graph/rateLimiter.ts b/src/graph/rateLimiter.ts index 85d8079..76554ba 100644 --- a/src/graph/rateLimiter.ts +++ b/src/graph/rateLimiter.ts @@ -1,20 +1,32 @@ import pLimit from 'p-limit' -export type GraphRequestPriority = 'high' | 'low' +export type RequestPriority = 'high' | 'low' +export type GraphRequestPriority = RequestPriority export interface GraphFetchOptions { priority?: GraphRequestPriority maxRetries?: number } -type ResolvedGraphFetchOptions = Required +export interface RateLimitedFetchOptions { + priority?: RequestPriority + maxRetries?: number + scope?: string + addPriorityHeader?: boolean + retryNetworkErrors?: boolean + globalBackoff?: boolean +} + +type ResolvedRateLimitedFetchOptions = Required -interface GraphRateLimitContext { +interface RateLimitContext { input: RequestInfo | URL method?: string - priority: GraphRequestPriority + priority: RequestPriority retry: number maxRetries: number + scope: string + globalBackoff: boolean } const DEFAULT_MAX_RETRIES = 2 @@ -29,6 +41,7 @@ const FALLBACK_RETRY_MAX_DELAY_MS = 180_000 const LOW_PRIORITY_CONCURRENCY = 2 let globalBackoffUntil = 0 +let highPriorityBackoffUntil = 0 let lowPriorityBackoffUntil = 0 const lowPriorityLimit = pLimit(LOW_PRIORITY_CONCURRENCY) @@ -80,14 +93,14 @@ const getRequestUrl = (input: RequestInfo | URL) => { return input.url } -const logGraphRateLimit = ( +const logRateLimit = ( message: string, - context: GraphRateLimitContext, + context: RateLimitContext, headers: Headers, status: number, delayMs: number, ) => { - console.warn(`[Graph] ${message}`, { + console.warn(`[${context.scope}] ${message}`, { status, method: context.method ?? 'GET', url: getRequestUrl(context.input), @@ -112,13 +125,13 @@ const sleep = (ms: number, signal?: AbortSignal): Promise => { return } - const timeout = window.setTimeout(() => { + const timeout = globalThis.setTimeout(() => { signal?.removeEventListener('abort', handleAbort) resolve() }, ms) const handleAbort = () => { - window.clearTimeout(timeout) + globalThis.clearTimeout(timeout) reject(signal?.reason ?? new DOMException('Aborted', 'AbortError')) } @@ -126,16 +139,16 @@ const sleep = (ms: number, signal?: AbortSignal): Promise => { }) } -const waitForBackoff = async (priority: GraphRequestPriority, signal?: AbortSignal) => { +const waitForBackoff = async (priority: RequestPriority, signal?: AbortSignal) => { const now = Date.now() const backoffUntil = priority === 'low' ? Math.max(globalBackoffUntil, lowPriorityBackoffUntil) - : globalBackoffUntil + : Math.max(globalBackoffUntil, highPriorityBackoffUntil) await sleep(backoffUntil - now, signal) } -const updateBackoffFromHeaders = (headers: Headers, status: number, context: GraphRateLimitContext) => { +const updateBackoffFromHeaders = (headers: Headers, status: number, context: RateLimitContext) => { const now = Date.now() const retryDelayMs = getGraphRetryDelayMs(headers) const rateLimitRemainingHeader = headers.get('RateLimit-Remaining') @@ -147,8 +160,14 @@ const updateBackoffFromHeaders = (headers: Headers, status: number, context: Gra if (isGlobalBackoffTriggered) { const delayMs = retryDelayMs ?? getFallbackRetryDelayMs(context.retry) - globalBackoffUntil = Math.max(globalBackoffUntil, now + delayMs) - logGraphRateLimit('Rate limit triggered; backing off Graph requests.', context, headers, status, delayMs) + if (context.globalBackoff) { + globalBackoffUntil = Math.max(globalBackoffUntil, now + delayMs) + } else if (context.priority === 'low') { + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) + } else { + highPriorityBackoffUntil = Math.max(highPriorityBackoffUntil, now + delayMs) + } + logRateLimit('Rate limit triggered; backing off requests.', context, headers, status, delayMs) } // Microsoft Graph files/lists resources (drive, driveItem, etc.) use SharePoint limits: @@ -166,7 +185,7 @@ const updateBackoffFromHeaders = (headers: Headers, status: number, context: Gra const delayMs = retryDelayMs ?? FALLBACK_RETRY_BASE_DELAY_MS lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) if (!isGlobalBackoffTriggered) { - logGraphRateLimit('Rate limit quota is low; backing off low-priority Graph requests.', context, headers, status, delayMs) + logRateLimit('Rate limit quota is low; backing off low-priority requests.', context, headers, status, delayMs) hasLoggedLowPriorityBackoff = true } } @@ -176,53 +195,93 @@ const updateBackoffFromHeaders = (headers: Headers, status: number, context: Gra const delayMs = retryDelayMs ?? FALLBACK_RETRY_BASE_DELAY_MS lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) if (!isGlobalBackoffTriggered && !hasLoggedLowPriorityBackoff) { - logGraphRateLimit('Throttle limit is high; backing off low-priority Graph requests.', context, headers, status, delayMs) + logRateLimit('Throttle limit is high; backing off low-priority requests.', context, headers, status, delayMs) } } } -const runGraphFetch = async ( +const runRateLimitedFetch = async ( input: RequestInfo | URL, init: RequestInit | undefined, - options: ResolvedGraphFetchOptions, + options: ResolvedRateLimitedFetchOptions, ): Promise => { const signal = init?.signal ?? undefined for (let retry = 0; retry <= options.maxRetries; retry += 1) { await waitForBackoff(options.priority, signal) - const headers = new Headers(init?.headers) - - // Priority header docs: - // https://learn.microsoft.com/en-us/graph/throttling-limits - headers.set('x-ms-throttle-priority', options.priority === 'low' ? 'Low' : 'High') - - const response = await fetch(input, { - ...init, - headers, - }) - - updateBackoffFromHeaders(response.headers, response.status, { - input, - method: init?.method, - priority: options.priority, - retry, - maxRetries: options.maxRetries, - }) - - if (!THROTTLED_STATUS_CODES.has(response.status) || retry >= options.maxRetries) { - return response + const requestInit = options.addPriorityHeader + ? (() => { + const headers = new Headers(init?.headers) + // https://learn.microsoft.com/en-us/graph/throttling-limits + headers.set('x-ms-throttle-priority', options.priority === 'low' ? 'Low' : 'High') + return { ...init, headers } + })() + : init + + try { + const response = await fetch(input, requestInit) + updateBackoffFromHeaders(response.headers, response.status, { + input, + method: init?.method, + priority: options.priority, + retry, + maxRetries: options.maxRetries, + scope: options.scope, + globalBackoff: options.globalBackoff, + }) + + if (!THROTTLED_STATUS_CODES.has(response.status) || retry >= options.maxRetries) return response + } catch (error) { + if (signal?.aborted || !options.retryNetworkErrors || retry >= options.maxRetries) throw error + const delayMs = getFallbackRetryDelayMs(retry) + if (options.globalBackoff) { + globalBackoffUntil = Math.max(globalBackoffUntil, Date.now() + delayMs) + } else if (options.priority === 'low') { + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, Date.now() + delayMs) + } else { + highPriorityBackoffUntil = Math.max(highPriorityBackoffUntil, Date.now() + delayMs) + } + console.warn(`[${options.scope}] Request failed; backing off requests.`, { + url: getRequestUrl(input), + priority: options.priority, + retry, + maxRetries: options.maxRetries, + delayMs, + error, + }) } } - throw new Error('Unexpected Graph fetch retry state.') + throw new Error('Unexpected rate-limited fetch retry state.') } const enqueueLowPriorityFetch = ( input: RequestInfo | URL, init: RequestInit | undefined, - options: ResolvedGraphFetchOptions, -): Promise => lowPriorityLimit(() => runGraphFetch(input, init, options)) + options: ResolvedRateLimitedFetchOptions, +): Promise => lowPriorityLimit(() => runRateLimitedFetch(input, init, options)) + +export const rateLimitedFetch = ( + input: RequestInfo | URL, + init?: RequestInit, + options: RateLimitedFetchOptions = {}, +): Promise => { + const resolvedOptions: ResolvedRateLimitedFetchOptions = { + priority: options.priority ?? 'high', + maxRetries: options.maxRetries ?? DEFAULT_MAX_RETRIES, + scope: options.scope ?? 'External', + addPriorityHeader: options.addPriorityHeader ?? false, + retryNetworkErrors: options.retryNetworkErrors ?? false, + globalBackoff: options.globalBackoff ?? false, + } + + if (resolvedOptions.priority === 'low') { + return enqueueLowPriorityFetch(input, init, resolvedOptions) + } + + return runRateLimitedFetch(input, init, resolvedOptions) +} export const graphFetch = ( input: RequestInfo | URL, @@ -232,11 +291,11 @@ export const graphFetch = ( maxRetries = DEFAULT_MAX_RETRIES, }: GraphFetchOptions = {}, ): Promise => { - const options = { priority, maxRetries } - - if (priority === 'low') { - return enqueueLowPriorityFetch(input, init, options) - } - - return runGraphFetch(input, init, options) + return rateLimitedFetch(input, init, { + priority, + maxRetries, + scope: 'Graph', + addPriorityHeader: true, + globalBackoff: true, + }) } diff --git a/src/hooks/graph/useGraph.ts b/src/hooks/graph/useGraph.ts index 0af4a85..4ce14b5 100644 --- a/src/hooks/graph/useGraph.ts +++ b/src/hooks/graph/useGraph.ts @@ -36,9 +36,15 @@ const useGraph = ( return { value: remoteItems } } - const getFileData = async (id: string, path?: string[], signal?: AbortSignal, priority?: GraphRequestPriority) => { + const getFileData = async ( + id: string, + path?: string[], + signal?: AbortSignal, + priority?: GraphRequestPriority, + includeThumbnails = true, + ) => { const accessToken = await getAccessToken() - const response = await getFile(accessToken, id, path, signal, priority) + const response = await getFile(accessToken, id, path, signal, priority, includeThumbnails) return response } @@ -82,7 +88,6 @@ const useGraph = ( } } - return { getFilesData, getFileData, diff --git a/src/hooks/graph/useMetadataSync.ts b/src/hooks/graph/useMetadataSync.ts new file mode 100644 index 0000000..c12e16c --- /dev/null +++ b/src/hooks/graph/useMetadataSync.ts @@ -0,0 +1,109 @@ +import { useEffect, useRef } from 'react' +import useUser from './useUser' +import useDb from '../useDb' +import useGraph from './useGraph' +import { useMsal } from '@azure/msal-react' +import useFileNodeSyncStore from '@/store/useFileNodeSyncStore' +import useMetadataSyncStore from '@/store/useMetadataSyncStore' +import { graphAudioToMetadata } from '@/utils/graphAudioMetadata' +import type { FileNode } from '@/types/file' +import { getRangeMetadata } from '@/utils/rangeMetadata' +import { RangeRequestError } from '@/utils/rangeReader' +import pLimit from 'p-limit' + +const MAX_ATTEMPTS = 3 +const RETRY_BASE_DELAY_MS = 1_000 +const metadataTaskLimit = pLimit(2) + +const wait = (ms: number) => new Promise(resolve => window.setTimeout(resolve, ms)) + +const useMetadataSync = () => { + const { instance } = useMsal() + const { account } = useUser() + const db = useDb(account) + const { getFileData } = useGraph(instance, account) + const fileSyncStatus = useFileNodeSyncStore.use.status() + const start = useMetadataSyncStore.use.start() + const finish = useMetadataSyncStore.use.finish() + const running = useRef(false) + + useEffect(() => { + if (!db || !account || fileSyncStatus !== 'success' || running.current) return + + const fetchWithRetry = async (node: FileNode) => { + let lastError: unknown + + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) { + try { + return await getFileData(node.id, undefined, undefined, 'low', false) + } catch (error) { + lastError = error + if (attempt < MAX_ATTEMPTS - 1) { + await wait(RETRY_BASE_DELAY_MS * (2 ** attempt)) + } + } + } + + throw lastError + } + + const run = async () => { + running.current = true + + try { + const nodes = await db.nodes + .where('metadataState') + .anyOf(['pending', 'failed']) + .and(node => node.type === 'audio') + .toArray() + + if (nodes.length === 0) return + + start() + + await Promise.all(nodes.map(node => metadataTaskLimit(async () => { + try { + const remoteItem = await fetchWithRetry(node) + const graphTitle = typeof remoteItem.audio?.title === 'string' && remoteItem.audio.title.trim() + let metadata + + if (graphTitle) { + metadata = graphAudioToMetadata(node, remoteItem.audio) + } else if (remoteItem['@microsoft.graph.downloadUrl']) { + try { + // Range requests must target the short-lived download URL. + // https://learn.microsoft.com/en-us/graph/api/driveitem-get-content?view=graph-rest-1.0#partial-range-downloads + metadata = await getRangeMetadata(node, remoteItem['@microsoft.graph.downloadUrl']) + } catch (error) { + if (error instanceof RangeRequestError) throw error + console.warn(`Falling back to filename metadata for ${node.name}:`, error) + metadata = graphAudioToMetadata(node) + } + } else { + throw new Error('No download URL returned for metadata extraction.') + } + + await db.transaction('rw', db.metadata, db.nodes, async () => { + await db.metadata.put(metadata) + await db.nodes.update(node.id, { metadataState: 'completed' }) + }) + } catch (error) { + console.error(`Failed to get metadata for ${node.name}:`, error) + await db.nodes.update(node.id, { metadataState: 'failed' }) + } + }))) + } catch (error) { + console.error('Failed to start metadata sync:', error) + } finally { + running.current = false + finish() + } + } + + void run() + // Graph helpers close over the current account and are recreated on render. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [account, db, fileSyncStatus, finish, start]) +} + +export default useMetadataSync diff --git a/src/hooks/player/useMetaData.ts b/src/hooks/player/useMetaData.ts index 8d9ba70..629977e 100644 --- a/src/hooks/player/useMetaData.ts +++ b/src/hooks/player/useMetaData.ts @@ -69,12 +69,15 @@ const useMetaData = (url: string) => { (async () => { if (currentTrack && currentTrack.track.id && isAudio(currentTrack.track.name) && db && url) { const localMetaData = await db.metadata.get(currentTrack.track.id) - if (!localMetaData) { + if (!localMetaData || localMetaData.source !== 'stream') { console.log('Start get net metadata: ', currentTrack.track) const result = await getNetMetaData(currentTrack.track, url) if (result) { - await db.metadata.put(result.metaData) - await db.pictures.bulkPut(result.pictureData) + await db.transaction('rw', db.metadata, db.pictures, db.nodes, async () => { + await db.metadata.put(result.metaData) + await db.pictures.bulkPut(result.pictureData) + await db.nodes.update(currentTrack.track.id, { metadataState: 'completed' }) + }) updateMetadataUpdate() } } @@ -87,4 +90,4 @@ const useMetaData = (url: string) => { } -export default useMetaData \ No newline at end of file +export default useMetaData diff --git a/src/locales/en/messages.po b/src/locales/en/messages.po index 53f15ed..496cbb4 100644 --- a/src/locales/en/messages.po +++ b/src/locales/en/messages.po @@ -48,11 +48,11 @@ msgstr "Add to play queue" msgid "Add to playlist" msgstr "Add to playlist" -#: src/pages/Library/Library.tsx:61 +#: src/pages/Library/Library.tsx:64 msgid "Album" msgstr "Album" -#: src/pages/Library/Library.tsx:70 +#: src/pages/Library/Library.tsx:73 msgid "Artist" msgstr "Artist" @@ -127,7 +127,7 @@ msgstr "Descending" msgid "Enter new title" msgstr "Enter new title" -#: src/pages/Library/Library.tsx:109 +#: src/pages/Library/Library.tsx:115 msgid "Error" msgstr "Error" @@ -135,7 +135,7 @@ msgstr "Error" msgid "Files" msgstr "Files" -#: src/pages/Library/Library.tsx:88 +#: src/pages/Library/Library.tsx:91 msgid "Folder" msgstr "Folder" @@ -160,7 +160,7 @@ msgstr "HD thumbnails" msgid "History" msgstr "History" -#: src/pages/Library/Library.tsx:107 +#: src/pages/Library/Library.tsx:113 msgid "Idle" msgstr "Idle" @@ -177,7 +177,7 @@ msgstr "Library" msgid "Library folder" msgstr "Library folder" -#: src/pages/Library/Library.tsx:120 +#: src/pages/Library/Library.tsx:126 msgid "Library folder not set" msgstr "Library folder not set" @@ -326,7 +326,7 @@ msgstr "Sign out" msgid "Size" msgstr "Size" -#: src/pages/Library/Library.tsx:79 +#: src/pages/Library/Library.tsx:82 msgid "Song" msgstr "Song" @@ -339,7 +339,7 @@ msgstr "Switch fullscreen" msgid "Switch theme" msgstr "Switch theme" -#: src/pages/Library/Library.tsx:108 +#: src/pages/Library/Library.tsx:114 msgid "Syncing" msgstr "Syncing" diff --git a/src/locales/zh-CN/messages.po b/src/locales/zh-CN/messages.po index cfc7160..48a07c3 100644 --- a/src/locales/zh-CN/messages.po +++ b/src/locales/zh-CN/messages.po @@ -48,11 +48,11 @@ msgstr "添加到播放队列" msgid "Add to playlist" msgstr "添加到播放列表" -#: src/pages/Library/Library.tsx:61 +#: src/pages/Library/Library.tsx:64 msgid "Album" msgstr "专辑" -#: src/pages/Library/Library.tsx:70 +#: src/pages/Library/Library.tsx:73 msgid "Artist" msgstr "艺术家" @@ -127,7 +127,7 @@ msgstr "倒序" msgid "Enter new title" msgstr "输入新标题" -#: src/pages/Library/Library.tsx:109 +#: src/pages/Library/Library.tsx:115 msgid "Error" msgstr "错误" @@ -135,7 +135,7 @@ msgstr "错误" msgid "Files" msgstr "文件" -#: src/pages/Library/Library.tsx:88 +#: src/pages/Library/Library.tsx:91 msgid "Folder" msgstr "文件夹" @@ -160,7 +160,7 @@ msgstr "高清缩略图" msgid "History" msgstr "历史" -#: src/pages/Library/Library.tsx:107 +#: src/pages/Library/Library.tsx:113 msgid "Idle" msgstr "空闲" @@ -177,7 +177,7 @@ msgstr "媒体库" msgid "Library folder" msgstr "媒体库文件夹" -#: src/pages/Library/Library.tsx:120 +#: src/pages/Library/Library.tsx:126 msgid "Library folder not set" msgstr "未设置媒体库文件夹" @@ -326,7 +326,7 @@ msgstr "注销" msgid "Size" msgstr "大小" -#: src/pages/Library/Library.tsx:79 +#: src/pages/Library/Library.tsx:82 msgid "Song" msgstr "歌曲" @@ -339,7 +339,7 @@ msgstr "切换全屏" msgid "Switch theme" msgstr "切换主题" -#: src/pages/Library/Library.tsx:108 +#: src/pages/Library/Library.tsx:114 msgid "Syncing" msgstr "同步中" diff --git a/src/pages/Library/Library.tsx b/src/pages/Library/Library.tsx index 6afe608..a99b1f5 100644 --- a/src/pages/Library/Library.tsx +++ b/src/pages/Library/Library.tsx @@ -11,6 +11,7 @@ import Loading from '../Loading' import { useMemo } from 'react' import { Link, Outlet, useLocation } from 'react-router-dom' import useFileNodeSyncStore from '@/store/useFileNodeSyncStore' +import useMetadataSyncStore from '@/store/useMetadataSyncStore' import SetLibraryFolderDialog from '@/components/Dialog/SetLibraryFolderDialog' const Library = () => { @@ -25,6 +26,7 @@ const Library = () => { const status = useFileNodeSyncStore.use.status() const error = useFileNodeSyncStore.use.error() + const metadataStatus = useMetadataSyncStore.use.status() const isLoadingSettings = settings === undefined @@ -51,9 +53,10 @@ const Library = () => { status === 'success' ? - + } @@ -92,6 +95,9 @@ const Library = () => { value="/library/folders" /> + + {metadataStatus === 'fetching' && } + @@ -123,4 +129,4 @@ const Library = () => { ) } -export default Library \ No newline at end of file +export default Library diff --git a/src/store/useMetadataSyncStore.ts b/src/store/useMetadataSyncStore.ts new file mode 100644 index 0000000..ef92e5c --- /dev/null +++ b/src/store/useMetadataSyncStore.ts @@ -0,0 +1,20 @@ +import { create } from 'zustand' +import createSelectors from './createSelectors' + +type MetadataSyncStatus = 'idle' | 'fetching' + +interface MetadataSyncStore { + status: MetadataSyncStatus + start: () => void + finish: () => void +} + +const useMetadataSyncStoreBase = create((set) => ({ + status: 'idle', + start: () => set({ status: 'fetching' }), + finish: () => set({ status: 'idle' }), +})) + +const useMetadataSyncStore = createSelectors(useMetadataSyncStoreBase) + +export default useMetadataSyncStore diff --git a/src/types/MetaData.ts b/src/types/MetaData.ts index e6d2020..ff05bfa 100644 --- a/src/types/MetaData.ts +++ b/src/types/MetaData.ts @@ -11,9 +11,10 @@ export interface PicutreData { export interface MetaData { id: string + source?: 'graph' | 'range' | 'filename' | 'stream' common: Omit & { albumartists?: string[] picture?: Picture[] } format: IAudioMetadata['format'] -} \ No newline at end of file +} diff --git a/src/types/file.ts b/src/types/file.ts index df05696..232ce41 100644 --- a/src/types/file.ts +++ b/src/types/file.ts @@ -9,6 +9,27 @@ export interface DeltaResponse { value: RemoteItem[] } +// The audio facet is populated by Microsoft Graph only for OneDrive Personal. +// https://learn.microsoft.com/en-us/graph/api/resources/audio?view=graph-rest-1.0 +export interface AudioDetails { + album?: string + albumArtist?: string + artist?: string + bitrate?: number + composers?: string + copyright?: string + disc?: number + discCount?: number + duration?: number + genre?: string + hasDrm?: boolean + isVariableBitrate?: boolean + title?: string + track?: number + trackCount?: number + year?: number +} + export interface FileDetails { mimeType: string hashes?: { @@ -74,6 +95,7 @@ export interface RemoteItem { file?: FileDetails folder?: FolderDetails + audio?: AudioDetails parentReference: ParentReference @@ -117,4 +139,4 @@ export interface Track { path: string[] size: number cTag?: string -} \ No newline at end of file +} diff --git a/src/utils/getNetMetaData.ts b/src/utils/getNetMetaData.ts index 292cfba..3171564 100644 --- a/src/utils/getNetMetaData.ts +++ b/src/utils/getNetMetaData.ts @@ -1,6 +1,7 @@ import { FileNode, Track } from '@/types/file' import { MetaData, Picture, PicutreData } from '@/types/metaData' import { IPicture, parseWebStream } from 'music-metadata' +import { rateLimitedFetch } from '@/graph/rateLimiter' export const compressImage = async (image: IPicture): Promise => { const blob = new Blob([image.data as unknown as ArrayBuffer], { type: image.format }) @@ -67,7 +68,10 @@ const getSha256 = async (data: Uint8Array): Promise => { const getNetMetaData = async (file: FileNode | Track, url: string): Promise<{ metaData: MetaData, pictureData: PicutreData[] } | null> => { try { - const response = await fetch(url) + const response = await rateLimitedFetch(url, undefined, { + scope: 'Content', + retryNetworkErrors: true, + }) if (response.body === null) { return null @@ -122,6 +126,7 @@ const getNetMetaData = async (file: FileNode | Track, url: string): Promise<{ me const metaData: MetaData = { id: file.id, + source: 'stream', common: { ...metadata.common, title: metadata.common.title.trim(), diff --git a/src/utils/graphAudioMetadata.ts b/src/utils/graphAudioMetadata.ts new file mode 100644 index 0000000..81fc56f --- /dev/null +++ b/src/utils/graphAudioMetadata.ts @@ -0,0 +1,47 @@ +import { AudioDetails, FileNode } from '@/types/file' +import { MetaData } from '@/types/metaData' + +const filenameTitle = (name: string) => name.replace(/\.[^.]+$/, '').trim() || name +const clean = (value?: unknown) => typeof value === 'string' ? value.trim() || undefined : undefined +const positive = (value?: number) => typeof value === 'number' && Number.isFinite(value) && value > 0 + ? value + : undefined + +export const graphAudioToMetadata = (file: FileNode, audio?: AudioDetails): MetaData => { + const graphTitle = clean(audio?.title) + const artist = clean(audio?.artist) + const albumartist = clean(audio?.albumArtist) + const album = clean(audio?.album) + const genre = clean(audio?.genre) + const composer = clean(audio?.composers) + const track = positive(audio?.track) + const trackCount = positive(audio?.trackCount) + const disk = positive(audio?.disc) + const diskCount = positive(audio?.discCount) + const duration = positive(audio?.duration) + const bitrate = positive(audio?.bitrate) + + return { + id: file.id, + source: graphTitle ? 'graph' : 'filename', + format: { + duration: duration === undefined ? undefined : duration / 1000, + bitrate: bitrate === undefined ? undefined : bitrate * 1000, + }, + native: {}, + quality: { warnings: [] }, + common: { + title: graphTitle ?? filenameTitle(file.name), + artist, + artists: artist ? [artist] : undefined, + albumartist, + album, + composer: composer ? [composer] : undefined, + genre: genre ? [genre] : undefined, + year: positive(audio?.year), + track: { no: track, of: trackCount }, + disk: { no: disk, of: diskCount }, + copyright: clean(audio?.copyright) ? [clean(audio?.copyright)!] : undefined, + }, + } +} diff --git a/src/utils/rangeMetadata.test.ts b/src/utils/rangeMetadata.test.ts new file mode 100644 index 0000000..77ad102 --- /dev/null +++ b/src/utils/rangeMetadata.test.ts @@ -0,0 +1,132 @@ +import { afterEach, describe, it } from 'node:test' +import assert from 'node:assert/strict' +import { getRangeMetadata } from './rangeMetadata.ts' +import { HttpRangeReader, RangeBudgetExceededError, RangeNotSupportedError } from './rangeReader.ts' +import type { FileNode } from '@/types/file' + +const originalFetch = globalThis.fetch +const concat = (...parts: Uint8Array[]) => { + const result = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)) + let offset = 0 + for (const part of parts) { + result.set(part, offset) + offset += part.length + } + return result +} + +const ascii = (value: string) => new TextEncoder().encode(value) +const u32be = (value: number) => new Uint8Array([(value >>> 24) & 255, (value >>> 16) & 255, (value >>> 8) & 255, value & 255]) +const u32le = (value: number) => new Uint8Array([value & 255, (value >>> 8) & 255, (value >>> 16) & 255, (value >>> 24) & 255]) +const syncSafe = (value: number) => new Uint8Array([(value >>> 21) & 127, (value >>> 14) & 127, (value >>> 7) & 127, value & 127]) +const frame = (id: string, payload: Uint8Array) => concat(ascii(id), u32be(payload.length), new Uint8Array(2), payload) +const flacBlock = (type: number, payload: Uint8Array, last = false) => concat( + new Uint8Array([(last ? 0x80 : 0) | type, (payload.length >>> 16) & 255, (payload.length >>> 8) & 255, payload.length & 255]), + payload, +) + +const mockRanges = (file: Uint8Array, status = 206) => { + const requested: Array<[number, number]> = [] + globalThis.fetch = async (_input: string | URL | Request, init?: RequestInit) => { + const range = new Headers(init?.headers).get('Range')! + const match = range.match(/^bytes=(\d+)-(\d+)$/)! + const start = Number(match[1]) + const end = Number(match[2]) + requested.push([start, end]) + if (status !== 206) return new Response(file, { status }) + return new Response(file.slice(start, end + 1), { + status: 206, + headers: { 'Content-Range': `bytes ${start}-${end}/${file.length}` }, + }) + } + return requested +} + +afterEach(() => { globalThis.fetch = originalFetch }) + +describe('HttpRangeReader', () => { + it('rejects a server that ignores Range', async () => { + mockRanges(new Uint8Array(32), 200) + await assert.rejects(() => new HttpRangeReader('test').read(0, 4), RangeNotSupportedError) + }) + + it('enforces the cumulative byte budget before requesting excess bytes', async () => { + const file = new Uint8Array(1024 * 1024 + 1) + const requested = mockRanges(file) + const reader = new HttpRangeReader('test') + await reader.read(0, 1024 * 1024) + await assert.rejects(() => reader.read(1024 * 1024, 1), RangeBudgetExceededError) + assert.equal(requested.length, 16) + }) + + it('accepts and caches an aligned partial response covering the requested range', async () => { + const file = new Uint8Array(1024).map((_, index) => index % 256) + let requests = 0 + globalThis.fetch = async () => { + requests += 1 + return new Response(file.slice(0, 512), { + status: 206, + headers: { + 'Content-Range': `bytes 0-511/${file.length}`, + 'Content-Length': '512', + }, + }) + } + const reader = new HttpRangeReader('test') + + assert.deepEqual(await reader.read(100, 4), file.slice(100, 104)) + assert.deepEqual(await reader.read(200, 4), file.slice(200, 204)) + assert.equal(requests, 1) + assert.equal(reader.bytesTransferred, 512) + }) + + it('accepts a 206 response when CORS hides Content-Range', async () => { + const file = new Uint8Array([10, 20, 30, 40]) + globalThis.fetch = async () => new Response(file, { + status: 206, + headers: { 'Content-Length': String(file.length) }, + }) + + const reader = new HttpRangeReader('test') + assert.deepEqual(await reader.read(0, 4), file) + assert.equal(reader.bytesTransferred, 4) + }) +}) + +describe('getRangeMetadata', () => { + it('extracts ID3 text frames without requesting APIC payload bytes', async () => { + const title = frame('TIT2', concat(new Uint8Array([3]), ascii('Range title'))) + const picture = frame('APIC', new Uint8Array(200_000).fill(7)) + const artist = frame('TPE1', concat(new Uint8Array([3]), ascii('Range artist'))) + const body = concat(title, picture, artist) + const file = concat(ascii('ID3'), new Uint8Array([3, 0, 0]), syncSafe(body.length), body, new Uint8Array(100)) + const requested = mockRanges(file) + const pictureStart = 10 + title.length + 10 + const pictureEnd = pictureStart + 200_000 - 1 + const node = { id: '1', name: 'song.mp3', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'Range title') + assert.equal(metadata.common.artist, 'Range artist') + assert.equal(metadata.source, 'range') + assert.equal(requested.some(([start, end]) => start <= pictureEnd && end >= pictureStart), false) + }) + + it('skips a FLAC PICTURE block and reads following Vorbis comments', async () => { + const picture = new Uint8Array(200_000).fill(9) + const vendor = ascii('test') + const comment = ascii('TITLE=FLAC title') + const comments = concat(u32le(vendor.length), vendor, u32le(1), u32le(comment.length), comment) + const file = concat(ascii('fLaC'), flacBlock(6, picture), flacBlock(4, comments, true)) + const requested = mockRanges(file) + const pictureStart = 8 + const pictureEnd = pictureStart + picture.length - 1 + const node = { id: '2', name: 'song.flac', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'FLAC title') + assert.equal(requested.some(([start, end]) => start <= pictureEnd && end >= pictureStart), false) + }) +}) diff --git a/src/utils/rangeMetadata.ts b/src/utils/rangeMetadata.ts new file mode 100644 index 0000000..f74d874 --- /dev/null +++ b/src/utils/rangeMetadata.ts @@ -0,0 +1,371 @@ +import type { FileNode } from '@/types/file' +import type { MetaData } from '@/types/metaData' +import { HttpRangeReader } from './rangeReader.ts' + +type TextTags = Partial<{ + title: string + artist: string + albumartist: string + album: string + composer: string + genre: string + year: number + track: number + trackTotal: number + disk: number + diskTotal: number + copyright: string +}> + +const decoder = new TextDecoder() +const ascii = (bytes: Uint8Array) => String.fromCharCode(...bytes) +const u32be = (b: Uint8Array, o = 0) => new DataView(b.buffer, b.byteOffset, b.byteLength).getUint32(o) +const u32le = (b: Uint8Array, o = 0) => new DataView(b.buffer, b.byteOffset, b.byteLength).getUint32(o, true) +const syncSafe = (b: Uint8Array, o = 0) => ((b[o] & 0x7f) << 21) | ((b[o + 1] & 0x7f) << 14) | ((b[o + 2] & 0x7f) << 7) | (b[o + 3] & 0x7f) +const trim = (value: string) => value.replace(/\0/g, '').trim() +const numberPair = (value: string) => { + const [no, of] = value.split('/').map(item => Number.parseInt(item, 10)) + return { no: Number.isFinite(no) ? no : undefined, of: Number.isFinite(of) ? of : undefined } +} + +const decodeText = (bytes: Uint8Array, encoding = 3) => { + if (bytes.length === 0) return '' + if (encoding === 0) return trim(new TextDecoder('windows-1252').decode(bytes)) + if (encoding === 3) return trim(decoder.decode(bytes)) + if (encoding === 2) return trim(new TextDecoder('utf-16be').decode(bytes)) + if (bytes[0] === 0xff && bytes[1] === 0xfe) return trim(new TextDecoder('utf-16le').decode(bytes.subarray(2))) + if (bytes[0] === 0xfe && bytes[1] === 0xff) return trim(new TextDecoder('utf-16be').decode(bytes.subarray(2))) + return trim(new TextDecoder('utf-16le').decode(bytes)) +} + +const applyTag = (tags: TextTags, key: string, value: string) => { + const normalized = key.toUpperCase() + if (!value) return + if (['TIT2', 'TT2', 'TITLE', '©NAM', 'INAM'].includes(normalized)) tags.title = value + else if (['TPE1', 'TP1', 'ARTIST', '©ART', 'IART'].includes(normalized)) tags.artist = value + else if (['TPE2', 'TP2', 'ALBUMARTIST', 'ALBUM ARTIST', 'AART'].includes(normalized)) tags.albumartist = value + else if (['TALB', 'TAL', 'ALBUM', '©ALB', 'IPRD'].includes(normalized)) tags.album = value + else if (['TCOM', 'TCM', 'COMPOSER', '©WRT', 'IWRI'].includes(normalized)) tags.composer = value + else if (['TCON', 'TCO', 'GENRE', '©GEN', 'IGNR'].includes(normalized)) tags.genre = value + else if (['TYER', 'TYE', 'TDRC', 'DATE', 'YEAR', '©DAY', 'ICRD'].includes(normalized)) tags.year = Number.parseInt(value, 10) || undefined + else if (['TRCK', 'TRK', 'TRACKNUMBER'].includes(normalized)) Object.assign(tags, { track: numberPair(value).no, trackTotal: numberPair(value).of }) + else if (normalized === 'TRACKTOTAL' || normalized === 'TOTALTRACKS') tags.trackTotal = Number.parseInt(value, 10) || undefined + else if (['TPOS', 'TPA', 'DISCNUMBER'].includes(normalized)) Object.assign(tags, { disk: numberPair(value).no, diskTotal: numberPair(value).of }) + else if (normalized === 'DISCTOTAL' || normalized === 'TOTALDISCS') tags.diskTotal = Number.parseInt(value, 10) || undefined + else if (['TCOP', 'TCR', 'COPYRIGHT', 'ICOP'].includes(normalized)) tags.copyright = value +} + +const parseId3 = async (reader: HttpRangeReader, base = 0): Promise => { + const tags: TextTags = {} + const header = await reader.read(base, 10) + if (ascii(header.subarray(0, 3)) !== 'ID3') return tags + const version = header[3] + const end = base + 10 + syncSafe(header, 6) + let offset = base + 10 + + while (offset + 6 <= end) { + const frameHeaderLength = version === 2 ? 6 : 10 + if (offset + frameHeaderLength > end) break + const frameHeader = await reader.read(offset, frameHeaderLength) + const idLength = version === 2 ? 3 : 4 + const id = ascii(frameHeader.subarray(0, idLength)) + if (!new RegExp(`^[A-Z0-9]{${idLength}}$`).test(id)) break + const size = version === 2 + ? (frameHeader[3] << 16) | (frameHeader[4] << 8) | frameHeader[5] + : version === 4 ? syncSafe(frameHeader, 4) : u32be(frameHeader, 4) + offset += frameHeaderLength + if (size <= 0 || offset + size > end) break + + if (id.startsWith('T') && id !== 'TXXX') { + const data = await reader.read(offset, size) + applyTag(tags, id, decodeText(data.subarray(1), data[0])) + } else if (id === 'COMM' && size > 4) { + const data = await reader.read(offset, size) + const encoding = data[0] + const separator = encoding === 0 || encoding === 3 ? 1 : 2 + let pos = 4 + while (pos + separator <= data.length && data.subarray(pos, pos + separator).some(byte => byte !== 0)) pos += separator + const value = decodeText(data.subarray(pos + separator), encoding) + if (value && !tags.title) applyTag(tags, 'COMMENT', value) + } + offset += size + } + + return tags +} + +const parseId3v1 = async (reader: HttpRangeReader, fileSize: number): Promise => { + if (fileSize < 128) return {} + const data = await reader.read(fileSize - 128, 128) + if (ascii(data.subarray(0, 3)) !== 'TAG') return {} + const tags: TextTags = {} + applyTag(tags, 'TITLE', decodeText(data.subarray(3, 33), 0)) + applyTag(tags, 'ARTIST', decodeText(data.subarray(33, 63), 0)) + applyTag(tags, 'ALBUM', decodeText(data.subarray(63, 93), 0)) + applyTag(tags, 'YEAR', decodeText(data.subarray(93, 97), 0)) + if (data[125] === 0 && data[126] > 0) tags.track = data[126] + return tags +} + +const parseFlacComments = async (reader: HttpRangeReader, start: number, length: number): Promise => { + const tags: TextTags = {} + const end = start + length + let offset = start + const vendorLength = u32le(await reader.read(offset, 4)); offset += 4 + vendorLength + const count = u32le(await reader.read(offset, 4)); offset += 4 + + for (let index = 0; index < count && offset + 4 <= end; index += 1) { + const commentLength = u32le(await reader.read(offset, 4)); offset += 4 + if (offset + commentLength > end) throw new Error('Invalid FLAC comment length.') + const prefixLength = Math.min(commentLength, 128) + const prefix = await reader.read(offset, prefixLength) + const prefixText = decoder.decode(prefix) + const separator = prefixText.indexOf('=') + const key = separator < 0 ? '' : prefixText.slice(0, separator).toUpperCase() + + if (!['METADATA_BLOCK_PICTURE', 'COVERART', 'COVERARTMIME'].includes(key)) { + const rest = commentLength > prefixLength + ? await reader.read(offset + prefixLength, commentLength - prefixLength) + : new Uint8Array() + const value = decoder.decode(concatBytes(prefix, rest)) + const valueSeparator = value.indexOf('=') + if (valueSeparator > 0) applyTag(tags, value.slice(0, valueSeparator), trim(value.slice(valueSeparator + 1))) + } + offset += commentLength + } + return tags +} + +const parseFlac = async (reader: HttpRangeReader): Promise => { + let offset = 4 + while (true) { + const header = await reader.read(offset, 4) + const last = (header[0] & 0x80) !== 0 + const type = header[0] & 0x7f + const length = (header[1] << 16) | (header[2] << 8) | header[3] + offset += 4 + if (type === 4) return parseFlacComments(reader, offset, length) + offset += length + if (last) return {} + } +} + +type Atom = { name: string, start: number, dataStart: number, end: number } +const readAtom = async (reader: HttpRangeReader, start: number, parentEnd: number): Promise => { + if (start + 8 > parentEnd) return null + const header = await reader.read(start, 8) + let size = u32be(header) + const name = ascii(header.subarray(4, 8)) + let headerSize = 8 + if (size === 1) { + const large = await reader.read(start + 8, 8) + size = Number(new DataView(large.buffer, large.byteOffset).getBigUint64(0)) + headerSize = 16 + } else if (size === 0) size = parentEnd - start + if (size < headerSize || start + size > parentEnd) return null + return { name, start, dataStart: start + headerSize, end: start + size } +} + +const parseMp4 = async (reader: HttpRangeReader, fileSize: number): Promise => { + const tags: TextTags = {} + const containers = new Set(['moov', 'udta', 'meta', 'ilst']) + const walk = async (start: number, end: number, inIlst = false) => { + let offset = start + while (offset + 8 <= end) { + const atom = await readAtom(reader, offset, end) + if (!atom) break + if (containers.has(atom.name)) { + await walk(atom.dataStart + (atom.name === 'meta' ? 4 : 0), atom.end, atom.name === 'ilst' || inIlst) + } else if (inIlst && atom.name !== 'covr') { + let childOffset = atom.dataStart + while (childOffset + 16 <= atom.end) { + const child = await readAtom(reader, childOffset, atom.end) + if (!child) break + if (child.name === 'data' && child.end - child.dataStart >= 8) { + const payload = await reader.read(child.dataStart + 8, child.end - child.dataStart - 8) + if (atom.name === 'trkn' && payload.length >= 6) { + tags.track = (payload[2] << 8) | payload[3] + tags.trackTotal = (payload[4] << 8) | payload[5] + } else if (atom.name === 'disk' && payload.length >= 6) { + tags.disk = (payload[2] << 8) | payload[3] + tags.diskTotal = (payload[4] << 8) | payload[5] + } else applyTag(tags, atom.name, trim(decoder.decode(payload))) + } + childOffset = child.end + } + } + offset = atom.end + } + } + await walk(0, fileSize) + return tags +} + +const parseWav = async (reader: HttpRangeReader, fileSize: number): Promise => { + const tags: TextTags = {} + let offset = 12 + while (offset + 8 <= fileSize) { + const header = await reader.read(offset, 8) + const id = ascii(header.subarray(0, 4)) + const size = u32le(header, 4) + const dataStart = offset + 8 + if (id === 'ID3 ' || id === 'id3 ') Object.assign(tags, await parseId3(reader, dataStart)) + if (id === 'LIST' && size >= 4 && ascii(await reader.read(dataStart, 4)) === 'INFO') { + let itemOffset = dataStart + 4 + while (itemOffset + 8 <= dataStart + size) { + const itemHeader = await reader.read(itemOffset, 8) + const itemSize = u32le(itemHeader, 4) + applyTag(tags, ascii(itemHeader.subarray(0, 4)), trim(decoder.decode(await reader.read(itemOffset + 8, itemSize)))) + itemOffset += 8 + itemSize + (itemSize % 2) + } + } + offset = dataStart + size + (size % 2) + } + return tags +} + +type OggSegment = { offset: number, length: number } + +class OggPacketReader { + private position = 0 + private readonly reader: HttpRangeReader + private readonly segments: OggSegment[] + + constructor(reader: HttpRangeReader, segments: OggSegment[]) { + this.reader = reader + this.segments = segments + } + + async read(length: number): Promise { + const output = new Uint8Array(length) + let outputOffset = 0 + let logicalOffset = 0 + + for (const segment of this.segments) { + const segmentEnd = logicalOffset + segment.length + if (this.position < segmentEnd && outputOffset < length) { + const withinSegment = Math.max(0, this.position - logicalOffset) + const take = Math.min(segment.length - withinSegment, length - outputOffset) + output.set(await this.reader.read(segment.offset + withinSegment, take), outputOffset) + this.position += take + outputOffset += take + } + logicalOffset = segmentEnd + if (outputOffset === length) return output + } + throw new Error('Unexpected end of Ogg packet.') + } + + skip(length: number) { + this.position += length + } +} + +const parseOgg = async (reader: HttpRangeReader, fileSize: number): Promise => { + const packets: OggSegment[][] = [] + let currentPacket: OggSegment[] = [] + let offset = 0 + + while (offset + 27 <= fileSize && packets.length < 2) { + const header = await reader.read(offset, 27) + if (ascii(header.subarray(0, 4)) !== 'OggS') break + const segmentCount = header[26] + const lacing = await reader.read(offset + 27, segmentCount) + let payloadOffset = offset + 27 + segmentCount + + for (const segmentLength of lacing) { + currentPacket.push({ offset: payloadOffset, length: segmentLength }) + payloadOffset += segmentLength + if (segmentLength < 255) { + packets.push(currentPacket) + currentPacket = [] + } + } + offset = payloadOffset + } + + if (packets.length < 2) return {} + const packet = new OggPacketReader(reader, packets[1]) + const signature = await packet.read(8) + if (ascii(signature.subarray(0, 7)) === '\x03vorbis') packet.skip(-1) + else if (ascii(signature) !== 'OpusTags') return {} + + const vendorLength = u32le(await packet.read(4)) + packet.skip(vendorLength) + const count = u32le(await packet.read(4)) + const tags: TextTags = {} + + for (let index = 0; index < count; index += 1) { + const length = u32le(await packet.read(4)) + const prefixLength = Math.min(length, 128) + const prefix = await packet.read(prefixLength) + const prefixText = decoder.decode(prefix) + const separator = prefixText.indexOf('=') + const key = separator < 0 ? '' : prefixText.slice(0, separator).toUpperCase() + const remaining = length - prefixLength + + if (['METADATA_BLOCK_PICTURE', 'COVERART', 'COVERARTMIME'].includes(key)) { + packet.skip(remaining) + continue + } + + const rest = remaining > 0 ? await packet.read(remaining) : new Uint8Array() + const value = decoder.decode(new Uint8Array([...prefix, ...rest])) + const valueSeparator = value.indexOf('=') + if (valueSeparator > 0) applyTag(tags, value.slice(0, valueSeparator), trim(value.slice(valueSeparator + 1))) + } + + return tags +} + +const filenameTitle = (name: string) => name.replace(/\.[^.]+$/, '').trim() || name + +export const getRangeMetadata = async (file: FileNode, url: string): Promise => { + const reader = new HttpRangeReader(url) + const signature = await reader.read(0, Math.min(4, file.size)) + let tags: TextTags = {} + + if (ascii(signature.subarray(0, 3)) === 'ID3') tags = await parseId3(reader) + else if (ascii(signature.subarray(0, 4)) === 'fLaC') tags = await parseFlac(reader) + else if (ascii(signature.subarray(0, 4)) === 'OggS') tags = await parseOgg(reader, file.size) + else if (file.size >= 12) { + const extendedSignature = concatBytes(signature, await reader.read(4, 8)) + if (ascii(extendedSignature.subarray(4, 8)) === 'ftyp') tags = await parseMp4(reader, file.size) + else if (ascii(signature) === 'RIFF' && ascii(extendedSignature.subarray(8, 12)) === 'WAVE') tags = await parseWav(reader, file.size) + } + + if (/\.mp3$/i.test(file.name) && !tags.title) { + tags = { ...await parseId3v1(reader, file.size), ...tags } + } + + return { + id: file.id, + source: tags.title ? 'range' : 'filename', + format: {}, + native: {}, + quality: { warnings: [] }, + common: { + title: tags.title ?? filenameTitle(file.name), + artist: tags.artist, + artists: tags.artist ? [tags.artist] : undefined, + albumartist: tags.albumartist, + album: tags.album, + composer: tags.composer ? [tags.composer] : undefined, + genre: tags.genre ? [tags.genre] : undefined, + year: tags.year, + track: { no: tags.track, of: tags.trackTotal }, + disk: { no: tags.disk, of: tags.diskTotal }, + copyright: tags.copyright ? [tags.copyright] : undefined, + }, + } +} + +const concatBytes = (...parts: Uint8Array[]) => { + const output = new Uint8Array(parts.reduce((total, part) => total + part.length, 0)) + let offset = 0 + for (const part of parts) { + output.set(part, offset) + offset += part.length + } + return output +} diff --git a/src/utils/rangeReader.ts b/src/utils/rangeReader.ts new file mode 100644 index 0000000..4125c9a --- /dev/null +++ b/src/utils/rangeReader.ts @@ -0,0 +1,117 @@ +import { rateLimitedFetch } from '../graph/rateLimiter.ts' + +export class RangeBudgetExceededError extends Error { + constructor() { + super('Range download budget exceeded.') + } +} + +export class RangeNotSupportedError extends Error { + constructor(details?: string) { + super(`Server did not return a valid partial response${details ? `: ${details}` : '.'}`) + } +} + +export class RangeRequestError extends Error { + constructor(cause: unknown) { + super('Range request failed.', { cause }) + } +} + +const MAX_REQUEST_SIZE = 64 * 1024 + +export class HttpRangeReader { + private transferred = 0 + private readonly cache: Array<{ start: number, end: number, bytes: Uint8Array }> = [] + private readonly url: string + private readonly budget: number + + constructor(url: string, budget = 1024 * 1024) { + this.url = url + this.budget = budget + } + + get bytesTransferred() { + return this.transferred + } + + async read(offset: number, length: number): Promise { + if (offset < 0 || length < 0 || !Number.isSafeInteger(offset) || !Number.isSafeInteger(length)) { + throw new RangeError('Invalid byte range.') + } + + const result = new Uint8Array(length) + let written = 0 + + while (written < length) { + const chunkLength = Math.min(MAX_REQUEST_SIZE, length - written) + const start = offset + written + const end = start + chunkLength - 1 + const cached = this.cache.find(entry => entry.start <= start && entry.end >= end) + if (cached) { + result.set(cached.bytes.subarray(start - cached.start, end - cached.start + 1), written) + written += chunkLength + continue + } + + if (this.transferred + chunkLength > this.budget) throw new RangeBudgetExceededError() + const response = await this.fetchRange(start, end) + + if (response.status !== 206) { + await response.body?.cancel() + throw new RangeNotSupportedError(`status=${response.status}, requested=${start}-${end}`) + } + + const contentRange = response.headers.get('Content-Range') + const match = contentRange?.match(/^bytes (\d+)-(\d+)\/(\d+|\*)$/) + let responseStart = match ? Number(match[1]) : start + let responseEnd = match ? Number(match[2]) : end + if (match && (responseStart > start || responseEnd < end)) { + await response.body?.cancel() + throw new RangeNotSupportedError(`Content-Range=${contentRange ?? 'missing'}, requested=${start}-${end}`) + } + + const contentLengthHeader = response.headers.get('Content-Length') + const contentLength = contentLengthHeader === null ? undefined : Number(contentLengthHeader) + if (contentLength !== undefined && Number.isFinite(contentLength) && this.transferred + contentLength > this.budget) { + await response.body?.cancel() + throw new RangeBudgetExceededError() + } + + const bytes = new Uint8Array(await response.arrayBuffer()) + if (!match) { + if (bytes.byteLength < chunkLength) { + throw new RangeNotSupportedError(`received=${bytes.byteLength}, requested=${start}-${end}, Content-Range is not exposed`) + } + // OneDrive may hide Content-Range from browser code while returning a + // valid 206 response. Treat the body as starting at the requested byte. + responseStart = start + responseEnd = start + bytes.byteLength - 1 + } + const expectedLength = responseEnd - responseStart + 1 + if (bytes.byteLength !== expectedLength) { + throw new RangeNotSupportedError(`received=${bytes.byteLength}, Content-Range=${contentRange}`) + } + this.transferred += bytes.byteLength + if (this.transferred > this.budget) throw new RangeBudgetExceededError() + this.cache.push({ start: responseStart, end: responseEnd, bytes }) + + result.set(bytes.subarray(start - responseStart, end - responseStart + 1), written) + written += chunkLength + } + + return result + } + + private async fetchRange(start: number, end: number) { + try { + return await rateLimitedFetch( + this.url, + { headers: { Range: `bytes=${start}-${end}` } }, + { priority: 'low', scope: 'Content', retryNetworkErrors: true }, + ) + } catch (error) { + throw new RangeRequestError(error) + } + } +} diff --git a/src/utils/remote.ts b/src/utils/remote.ts index 245b2d6..25b4980 100644 --- a/src/utils/remote.ts +++ b/src/utils/remote.ts @@ -1,5 +1,6 @@ import { RemoteItem, FileNode } from '@/types/file' import checkFileType from './checkFileType' +import { rateLimitedFetch } from '@/graph/rateLimiter' /** * 根据 url 解析 json @@ -8,7 +9,10 @@ import checkFileType from './checkFileType' */ export const fetchJson = async (url: string) => { try { - const response = await fetch(url) + const response = await rateLimitedFetch(url, undefined, { + scope: 'External', + retryNetworkErrors: true, + }) const json = response.json() return json } catch (error) { @@ -60,4 +64,4 @@ export const sizeConv = (fileSize: number) => { : ((fileSize / 1024 / 1024) < 1024) ? `${(fileSize / 1024 / 1024).toFixed(2)} MB` : `${(fileSize / 1024 / 1024 / 1024).toFixed(2)} GB` -} \ No newline at end of file +} diff --git a/tsconfig.json b/tsconfig.json index 456caf1..4537626 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,7 @@ "noUnusedParameters": true, "noFallthroughCasesInSwitch": true, "allowSyntheticDefaultImports": true, + "allowImportingTsExtensions": true, "esModuleInterop": true, "noImplicitAny": true, "allowJs": false, @@ -35,4 +36,4 @@ "**/*.ts", "**/*.tsx", ] -} \ No newline at end of file +} From 3801a8c8771e363ec3a814c4bc79eab8497b70f5 Mon Sep 17 00:00:00 2001 From: Anlor Date: Sun, 12 Jul 2026 17:31:02 +0000 Subject: [PATCH 5/8] chore: update locale catalog references --- src/locales/en/messages.po | 2 +- src/locales/zh-CN/messages.po | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/locales/en/messages.po b/src/locales/en/messages.po index 496cbb4..3aba085 100644 --- a/src/locales/en/messages.po +++ b/src/locales/en/messages.po @@ -14,7 +14,7 @@ msgstr "" "Plural-Forms: \n" #. placeholder {0}: songs.length -#: src/pages/Library/AlbumDetail.tsx:136 +#: src/pages/Library/AlbumDetail.tsx:165 #: src/pages/Library/FolderDetail.tsx:90 msgid "{0} songs" msgstr "{0} songs" diff --git a/src/locales/zh-CN/messages.po b/src/locales/zh-CN/messages.po index 48a07c3..865e373 100644 --- a/src/locales/zh-CN/messages.po +++ b/src/locales/zh-CN/messages.po @@ -14,7 +14,7 @@ msgstr "" "Plural-Forms: \n" #. placeholder {0}: songs.length -#: src/pages/Library/AlbumDetail.tsx:136 +#: src/pages/Library/AlbumDetail.tsx:165 #: src/pages/Library/FolderDetail.tsx:90 msgid "{0} songs" msgstr "{0} 首歌曲" From 2a240fb11da530be57469ca64378d390c0715044 Mon Sep 17 00:00:00 2001 From: Anlor Date: Mon, 13 Jul 2026 15:37:56 +0000 Subject: [PATCH 6/8] fix: cancel stale metadata sync tasks --- src/hooks/graph/useMetadataSync.ts | 102 ++++++++++++++++++++++++----- src/utils/rangeMetadata.test.ts | 33 +++++++++- src/utils/rangeMetadata.ts | 10 +-- src/utils/rangeReader.ts | 15 ++++- 4 files changed, 137 insertions(+), 23 deletions(-) diff --git a/src/hooks/graph/useMetadataSync.ts b/src/hooks/graph/useMetadataSync.ts index c12e16c..4a25b9b 100644 --- a/src/hooks/graph/useMetadataSync.ts +++ b/src/hooks/graph/useMetadataSync.ts @@ -15,7 +15,28 @@ const MAX_ATTEMPTS = 3 const RETRY_BASE_DELAY_MS = 1_000 const metadataTaskLimit = pLimit(2) -const wait = (ms: number) => new Promise(resolve => window.setTimeout(resolve, ms)) +const getAbortReason = (signal: AbortSignal) => + signal.reason ?? new DOMException('Aborted', 'AbortError') + +const throwIfAborted = (signal: AbortSignal) => { + if (signal.aborted) throw getAbortReason(signal) +} + +const wait = (ms: number, signal: AbortSignal) => new Promise((resolve, reject) => { + throwIfAborted(signal) + + const timeout = window.setTimeout(() => { + signal.removeEventListener('abort', handleAbort) + resolve() + }, ms) + + const handleAbort = () => { + window.clearTimeout(timeout) + reject(getAbortReason(signal)) + } + + signal.addEventListener('abort', handleAbort, { once: true }) +}) const useMetadataSync = () => { const { instance } = useMsal() @@ -25,21 +46,31 @@ const useMetadataSync = () => { const fileSyncStatus = useFileNodeSyncStore.use.status() const start = useMetadataSyncStore.use.start() const finish = useMetadataSyncStore.use.finish() - const running = useRef(false) + const running = useRef(null) useEffect(() => { if (!db || !account || fileSyncStatus !== 'success' || running.current) return + const controller = new AbortController() + const { signal } = controller + const runToken = Symbol('metadata-sync') + let started = false + running.current = runToken + + const isCurrentRun = () => running.current === runToken && !signal.aborted + const fetchWithRetry = async (node: FileNode) => { let lastError: unknown for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) { + throwIfAborted(signal) try { - return await getFileData(node.id, undefined, undefined, 'low', false) + return await getFileData(node.id, undefined, signal, 'low', false) } catch (error) { + throwIfAborted(signal) lastError = error if (attempt < MAX_ATTEMPTS - 1) { - await wait(RETRY_BASE_DELAY_MS * (2 ** attempt)) + await wait(RETRY_BASE_DELAY_MS * (2 ** attempt), signal) } } } @@ -48,8 +79,6 @@ const useMetadataSync = () => { } const run = async () => { - running.current = true - try { const nodes = await db.nodes .where('metadataState') @@ -57,13 +86,18 @@ const useMetadataSync = () => { .and(node => node.type === 'audio') .toArray() - if (nodes.length === 0) return + if (nodes.length === 0 || !isCurrentRun()) return start() + started = true + + await Promise.allSettled(nodes.map(node => metadataTaskLimit(async () => { + if (!isCurrentRun()) return - await Promise.all(nodes.map(node => metadataTaskLimit(async () => { try { const remoteItem = await fetchWithRetry(node) + if (!isCurrentRun()) return + const graphTitle = typeof remoteItem.audio?.title === 'string' && remoteItem.audio.title.trim() let metadata @@ -73,8 +107,9 @@ const useMetadataSync = () => { try { // Range requests must target the short-lived download URL. // https://learn.microsoft.com/en-us/graph/api/driveitem-get-content?view=graph-rest-1.0#partial-range-downloads - metadata = await getRangeMetadata(node, remoteItem['@microsoft.graph.downloadUrl']) + metadata = await getRangeMetadata(node, remoteItem['@microsoft.graph.downloadUrl'], signal) } catch (error) { + if (!isCurrentRun()) return if (error instanceof RangeRequestError) throw error console.warn(`Falling back to filename metadata for ${node.name}:`, error) metadata = graphAudioToMetadata(node) @@ -83,24 +118,59 @@ const useMetadataSync = () => { throw new Error('No download URL returned for metadata extraction.') } - await db.transaction('rw', db.metadata, db.nodes, async () => { - await db.metadata.put(metadata) - await db.nodes.update(node.id, { metadataState: 'completed' }) + if (!isCurrentRun()) return + await db.transaction('rw', db.metadata, db.nodes, async transaction => { + const handleAbort = () => transaction.abort() + signal.addEventListener('abort', handleAbort, { once: true }) + try { + throwIfAborted(signal) + await db.metadata.put(metadata) + throwIfAborted(signal) + await db.nodes.update(node.id, { metadataState: 'completed' }) + } finally { + signal.removeEventListener('abort', handleAbort) + } }) } catch (error) { + if (!isCurrentRun()) return console.error(`Failed to get metadata for ${node.name}:`, error) - await db.nodes.update(node.id, { metadataState: 'failed' }) + try { + await db.transaction('rw', db.nodes, async transaction => { + const handleAbort = () => transaction.abort() + signal.addEventListener('abort', handleAbort, { once: true }) + try { + throwIfAborted(signal) + await db.nodes.update(node.id, { metadataState: 'failed' }) + } finally { + signal.removeEventListener('abort', handleAbort) + } + }) + } catch (dbError) { + if (isCurrentRun()) { + console.error(`Failed to update metadataState to failed for ${node.name}:`, dbError) + } + } } }))) } catch (error) { - console.error('Failed to start metadata sync:', error) + if (isCurrentRun()) console.error('Failed to start metadata sync:', error) } finally { - running.current = false - finish() + if (running.current === runToken) { + running.current = null + if (started) finish() + } } } void run() + + return () => { + controller.abort() + if (running.current === runToken) { + running.current = null + if (started) finish() + } + } // Graph helpers close over the current account and are recreated on render. // eslint-disable-next-line react-hooks/exhaustive-deps }, [account, db, fileSyncStatus, finish, start]) diff --git a/src/utils/rangeMetadata.test.ts b/src/utils/rangeMetadata.test.ts index 77ad102..68797b4 100644 --- a/src/utils/rangeMetadata.test.ts +++ b/src/utils/rangeMetadata.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, it } from 'node:test' import assert from 'node:assert/strict' import { getRangeMetadata } from './rangeMetadata.ts' -import { HttpRangeReader, RangeBudgetExceededError, RangeNotSupportedError } from './rangeReader.ts' +import { HttpRangeReader, RangeBudgetExceededError, RangeNotSupportedError, RangeRequestError } from './rangeReader.ts' import type { FileNode } from '@/types/file' const originalFetch = globalThis.fetch @@ -91,6 +91,37 @@ describe('HttpRangeReader', () => { assert.deepEqual(await reader.read(0, 4), file) assert.equal(reader.bytesTransferred, 4) }) + + it('does not issue a request when already aborted', async () => { + let requests = 0 + globalThis.fetch = async () => { + requests += 1 + return new Response(new Uint8Array(4), { status: 206 }) + } + const controller = new AbortController() + controller.abort() + + await assert.rejects(() => new HttpRangeReader('test', undefined, controller.signal).read(0, 4), { + name: 'AbortError', + }) + assert.equal(requests, 0) + }) + + it('passes cancellation to an in-flight range request', async () => { + let notifyStarted!: () => void + const started = new Promise(resolve => { notifyStarted = resolve }) + globalThis.fetch = async (_input, init) => new Promise((_resolve, reject) => { + notifyStarted() + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }) + }) + const controller = new AbortController() + const reading = new HttpRangeReader('test', undefined, controller.signal).read(0, 4) + + await started + controller.abort() + + await assert.rejects(() => reading, RangeRequestError) + }) }) describe('getRangeMetadata', () => { diff --git a/src/utils/rangeMetadata.ts b/src/utils/rangeMetadata.ts index f74d874..733270b 100644 --- a/src/utils/rangeMetadata.ts +++ b/src/utils/rangeMetadata.ts @@ -55,12 +55,14 @@ const applyTag = (tags: TextTags, key: string, value: string) => { else if (['TCOP', 'TCR', 'COPYRIGHT', 'ICOP'].includes(normalized)) tags.copyright = value } -const parseId3 = async (reader: HttpRangeReader, base = 0): Promise => { +const parseId3 = async (reader: HttpRangeReader, base = 0, maxEnd = Number.POSITIVE_INFINITY): Promise => { const tags: TextTags = {} + if (base + 10 > maxEnd) return tags const header = await reader.read(base, 10) if (ascii(header.subarray(0, 3)) !== 'ID3') return tags const version = header[3] const end = base + 10 + syncSafe(header, 6) + if (end > maxEnd) return tags let offset = base + 10 while (offset + 6 <= end) { @@ -320,12 +322,12 @@ const parseOgg = async (reader: HttpRangeReader, fileSize: number): Promise name.replace(/\.[^.]+$/, '').trim() || name -export const getRangeMetadata = async (file: FileNode, url: string): Promise => { - const reader = new HttpRangeReader(url) +export const getRangeMetadata = async (file: FileNode, url: string, signal?: AbortSignal): Promise => { + const reader = new HttpRangeReader(url, undefined, signal) const signature = await reader.read(0, Math.min(4, file.size)) let tags: TextTags = {} - if (ascii(signature.subarray(0, 3)) === 'ID3') tags = await parseId3(reader) + if (ascii(signature.subarray(0, 3)) === 'ID3') tags = await parseId3(reader, 0, file.size) else if (ascii(signature.subarray(0, 4)) === 'fLaC') tags = await parseFlac(reader) else if (ascii(signature.subarray(0, 4)) === 'OggS') tags = await parseOgg(reader, file.size) else if (file.size >= 12) { diff --git a/src/utils/rangeReader.ts b/src/utils/rangeReader.ts index 4125c9a..c69bd88 100644 --- a/src/utils/rangeReader.ts +++ b/src/utils/rangeReader.ts @@ -25,10 +25,12 @@ export class HttpRangeReader { private readonly cache: Array<{ start: number, end: number, bytes: Uint8Array }> = [] private readonly url: string private readonly budget: number + private readonly signal?: AbortSignal - constructor(url: string, budget = 1024 * 1024) { + constructor(url: string, budget = 1024 * 1024, signal?: AbortSignal) { this.url = url this.budget = budget + this.signal = signal } get bytesTransferred() { @@ -36,6 +38,8 @@ export class HttpRangeReader { } async read(offset: number, length: number): Promise { + this.throwIfAborted() + if (offset < 0 || length < 0 || !Number.isSafeInteger(offset) || !Number.isSafeInteger(length)) { throw new RangeError('Invalid byte range.') } @@ -44,6 +48,7 @@ export class HttpRangeReader { let written = 0 while (written < length) { + this.throwIfAborted() const chunkLength = Math.min(MAX_REQUEST_SIZE, length - written) const start = offset + written const end = start + chunkLength - 1 @@ -107,11 +112,17 @@ export class HttpRangeReader { try { return await rateLimitedFetch( this.url, - { headers: { Range: `bytes=${start}-${end}` } }, + { headers: { Range: `bytes=${start}-${end}` }, signal: this.signal }, { priority: 'low', scope: 'Content', retryNetworkErrors: true }, ) } catch (error) { throw new RangeRequestError(error) } } + + private throwIfAborted() { + if (this.signal?.aborted) { + throw this.signal.reason ?? new DOMException('Aborted', 'AbortError') + } + } } From 65eacfd8299a88a671e3a10c4ec998a87374e440 Mon Sep 17 00:00:00 2001 From: Anlor Date: Mon, 13 Jul 2026 15:38:29 +0000 Subject: [PATCH 7/8] fix: bound WAV metadata parsing --- src/utils/rangeMetadata.test.ts | 58 +++++++++++++++++++++++++++++++++ src/utils/rangeMetadata.ts | 15 ++++++--- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/utils/rangeMetadata.test.ts b/src/utils/rangeMetadata.test.ts index 68797b4..cddc727 100644 --- a/src/utils/rangeMetadata.test.ts +++ b/src/utils/rangeMetadata.test.ts @@ -24,6 +24,16 @@ const flacBlock = (type: number, payload: Uint8Array, last = false) => concat( new Uint8Array([(last ? 0x80 : 0) | type, (payload.length >>> 16) & 255, (payload.length >>> 8) & 255, payload.length & 255]), payload, ) +const wavChunk = (id: string, payload: Uint8Array) => concat( + ascii(id), + u32le(payload.length), + payload, + payload.length % 2 === 0 ? new Uint8Array() : new Uint8Array(1), +) +const wavFile = (...chunks: Uint8Array[]) => { + const body = concat(ascii('WAVE'), ...chunks) + return concat(ascii('RIFF'), u32le(body.length), body) +} const mockRanges = (file: Uint8Array, status = 206) => { const requested: Array<[number, number]> = [] @@ -160,4 +170,52 @@ describe('getRangeMetadata', () => { assert.equal(metadata.common.title, 'FLAC title') assert.equal(requested.some(([start, end]) => start <= pictureEnd && end >= pictureStart), false) }) + + it('extracts a title from a valid WAV LIST INFO chunk', async () => { + const title = ascii('WAV title\0') + const info = concat(ascii('INFO'), wavChunk('INAM', title)) + const file = wavFile(wavChunk('LIST', info)) + mockRanges(file) + const node = { id: '3', name: 'song.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'WAV title') + assert.equal(metadata.source, 'range') + }) + + it('does not read beyond a truncated outer WAV chunk', async () => { + const file = concat(ascii('RIFF'), u32le(16), ascii('WAVE'), ascii('LIST'), u32le(64)) + const requested = mockRanges(file) + const node = { id: '4', name: 'truncated.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'truncated') + assert.equal(requested.every(([, end]) => end < file.length), true) + }) + + it('does not read an INFO item payload beyond its LIST chunk', async () => { + const invalidItem = concat(ascii('INAM'), u32le(100)) + const file = wavFile(wavChunk('LIST', concat(ascii('INFO'), invalidItem))) + const requested = mockRanges(file) + const node = { id: '5', name: 'invalid-info.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'invalid-info') + assert.equal(requested.every(([, end]) => end < file.length), true) + }) + + it('bounds an embedded ID3 tag to its WAV chunk', async () => { + const oversizedId3 = concat(ascii('ID3'), new Uint8Array([3, 0, 0]), syncSafe(100)) + const file = wavFile(wavChunk('ID3 ', oversizedId3)) + const requested = mockRanges(file) + const node = { id: '6', name: 'invalid-id3.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'invalid-id3') + assert.equal(requested.every(([, end]) => end < file.length), true) + }) }) diff --git a/src/utils/rangeMetadata.ts b/src/utils/rangeMetadata.ts index 733270b..a2e050c 100644 --- a/src/utils/rangeMetadata.ts +++ b/src/utils/rangeMetadata.ts @@ -211,17 +211,22 @@ const parseWav = async (reader: HttpRangeReader, fileSize: number): Promise fileSize) break + if (id === 'ID3 ' || id === 'id3 ') Object.assign(tags, await parseId3(reader, dataStart, dataEnd)) if (id === 'LIST' && size >= 4 && ascii(await reader.read(dataStart, 4)) === 'INFO') { let itemOffset = dataStart + 4 - while (itemOffset + 8 <= dataStart + size) { + while (itemOffset + 8 <= dataEnd) { const itemHeader = await reader.read(itemOffset, 8) const itemSize = u32le(itemHeader, 4) - applyTag(tags, ascii(itemHeader.subarray(0, 4)), trim(decoder.decode(await reader.read(itemOffset + 8, itemSize)))) - itemOffset += 8 + itemSize + (itemSize % 2) + const itemDataStart = itemOffset + 8 + const itemDataEnd = itemDataStart + itemSize + if (itemDataEnd > dataEnd) break + applyTag(tags, ascii(itemHeader.subarray(0, 4)), trim(decoder.decode(await reader.read(itemDataStart, itemSize)))) + itemOffset = itemDataEnd + (itemSize % 2) } } - offset = dataStart + size + (size % 2) + offset = dataEnd + (size % 2) } return tags } From 10016b0364e8f7e2dffa3547c44ea1d195a4c542 Mon Sep 17 00:00:00 2001 From: Anlor Date: Mon, 13 Jul 2026 15:44:54 +0000 Subject: [PATCH 8/8] fix: improve performance --- src/utils/rangeMetadata.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/rangeMetadata.ts b/src/utils/rangeMetadata.ts index a2e050c..bcad74e 100644 --- a/src/utils/rangeMetadata.ts +++ b/src/utils/rangeMetadata.ts @@ -64,14 +64,14 @@ const parseId3 = async (reader: HttpRangeReader, base = 0, maxEnd = Number.POSIT const end = base + 10 + syncSafe(header, 6) if (end > maxEnd) return tags let offset = base + 10 + const frameHeaderLength = version === 2 ? 6 : 10 + const idLength = version === 2 ? 3 : 4 + const idPattern = version === 2 ? /^[A-Z0-9]{3}$/ : /^[A-Z0-9]{4}$/ - while (offset + 6 <= end) { - const frameHeaderLength = version === 2 ? 6 : 10 - if (offset + frameHeaderLength > end) break + while (offset + frameHeaderLength <= end) { const frameHeader = await reader.read(offset, frameHeaderLength) - const idLength = version === 2 ? 3 : 4 const id = ascii(frameHeader.subarray(0, idLength)) - if (!new RegExp(`^[A-Z0-9]{${idLength}}$`).test(id)) break + if (!idPattern.test(id)) break const size = version === 2 ? (frameHeader[3] << 16) | (frameHeader[4] << 8) | frameHeader[5] : version === 4 ? syncSafe(frameHeader, 4) : u32be(frameHeader, 4) @@ -317,7 +317,7 @@ const parseOgg = async (reader: HttpRangeReader, fileSize: number): Promise 0 ? await packet.read(remaining) : new Uint8Array() - const value = decoder.decode(new Uint8Array([...prefix, ...rest])) + const value = decoder.decode(concatBytes(prefix, rest)) const valueSeparator = value.indexOf('=') if (valueSeparator > 0) applyTag(tags, value.slice(0, valueSeparator), trim(value.slice(valueSeparator + 1))) }