diff --git a/package-lock.json b/package-lock.json index ae99a8f..7115970 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "franc-min": "^6.2.0", "idb-keyval": "^6.2.1", "music-metadata": "^11.9.0", + "p-limit": "^7.3.0", "pinyin-pro": "^3.27.0", "process": "^0.11.10", "react": "^18.3.1", @@ -9998,16 +9999,15 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "7.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", "license": "MIT", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.2.1" }, "engines": { - "node": ">=10" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -10029,6 +10029,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmmirror.com/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-retry": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", @@ -14053,13 +14082,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.2.2", + "resolved": "https://registry.npmmirror.com/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index 6002f0a..cc13650 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "dev": "npm run generate:dev && webpack serve", "lint": "npm run generate && eslint src --report-unused-disable-directives --max-warnings 0", "lint:dev": "npm run generate:dev && eslint src --report-unused-disable-directives --max-warnings 0", + "test": "node --experimental-strip-types --test src/utils/rangeMetadata.test.ts", "lingui": "lingui extract && lingui compile --typescript", "generate": "node generate-version-info.mjs && npm run lingui", "generate:dev": "node generate-version-info.mjs --dev && npm run lingui" @@ -34,6 +35,7 @@ "franc-min": "^6.2.0", "idb-keyval": "^6.2.1", "music-metadata": "^11.9.0", + "p-limit": "^7.3.0", "pinyin-pro": "^3.27.0", "process": "^0.11.10", "react": "^18.3.1", @@ -88,4 +90,4 @@ "webpack-merge": "^6.0.1", "workbox-webpack-plugin": "^7.3.0" } -} \ No newline at end of file +} diff --git a/src/App.tsx b/src/App.tsx index 2e988f5..8ab9478 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -20,6 +20,7 @@ import useFileNodeSyncStore from './store/useFileNodeSyncStore' import { useLiveQuery } from 'dexie-react-hooks' import useDb from './hooks/useDb' import useTitle from './hooks/ui/useTitle' +import useMetadataSync from './hooks/graph/useMetadataSync' const App = () => { useEnvironment() @@ -32,6 +33,7 @@ const App = () => { const { account } = useUser() useSync() useFileNodeSync() + useMetadataSync() const db = useDb(account) const libraryRootId = useLiveQuery(async () => (await db?.settings.get('settings'))?.libraryRootId, [db]) @@ -141,4 +143,4 @@ const App = () => { ) } -export default App \ No newline at end of file +export default App diff --git a/src/graph/graph.ts b/src/graph/graph.ts index e163eaa..6df2389 100644 --- a/src/graph/graph.ts +++ b/src/graph/graph.ts @@ -1,11 +1,13 @@ import { DeltaResponse, FileResponse, RemoteItem } from '@/types/file' import { graphConfig } from './authConfig' +import { graphFetch, GraphRequestPriority } from './rateLimiter' export async function getFiles( accessToken: string, id: string, path?: string[], nextLink?: string, + priority?: GraphRequestPriority, ): Promise { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -30,45 +32,54 @@ export async function getFiles( : `${graphConfig.graphMeEndpoint}/me/drive/root:/${encodeURIComponent(path.join('/'))}:/children?${params.toString()}` : `${graphConfig.graphMeEndpoint}/me/drive/items/${id}/children?${params.toString()}` - return fetch(nextLink || url, options) + return graphFetch(nextLink || url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } -export async function getFile( +const parseGraphResponse = async (response: Response): Promise => { + if (!response.ok) { + throw new Error(`Graph request failed with status ${response.status}.`) + } + + return response.json() as Promise +} + +const requestDriveItem = async ( accessToken: string, id: string, path?: string[], signal?: AbortSignal, -): Promise { - const headers = new Headers() - const bearer = `Bearer ${accessToken}` - - headers.append('Authorization', bearer) - - const options = { - method: 'GET', - headers: headers, - signal: signal, - } - - const queryParams = { - $expand: 'thumbnails' - } - - const params = new URLSearchParams(queryParams) + priority?: GraphRequestPriority, + includeThumbnails = false, +): Promise => { + const headers = new Headers({ Authorization: `Bearer ${accessToken}` }) + const query = includeThumbnails + ? `?${new URLSearchParams({ $expand: 'thumbnails' }).toString()}` + : '' const url = path - ? `${graphConfig.graphMeEndpoint}/me/drive/root:/${encodeURIComponent(path.join('/'))}?${params.toString()}` - : `${graphConfig.graphMeEndpoint}/me/drive/items/${id}?${params.toString()}` + ? `${graphConfig.graphMeEndpoint}/me/drive/root:/${encodeURIComponent(path.join('/'))}${query}` + : `${graphConfig.graphMeEndpoint}/me/drive/items/${id}${query}` + const response = await graphFetch(url, { method: 'GET', headers, signal }, { priority }) - return fetch(url, options) - .then(response => response.json()) - .catch(error => console.log(error)) + return parseGraphResponse(response) +} + +export async function getFile( + accessToken: string, + id: string, + path?: string[], + signal?: AbortSignal, + priority?: GraphRequestPriority, + includeThumbnails = true, +): Promise { + return requestDriveItem(accessToken, id, path, signal, priority, includeThumbnails) } export const getAppRootFiles = async ( accessToken: string, + priority?: GraphRequestPriority, ) => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -82,7 +93,7 @@ export const getAppRootFiles = async ( const url = `${graphConfig.graphMeEndpoint}/me/drive/special/approot/children` - return fetch(url, options) + return graphFetch(url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } @@ -91,6 +102,7 @@ export const uploadAppRootJson = async ( accessToken: string, fileName: string, fileContent: BodyInit, + priority?: GraphRequestPriority, ) => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -106,7 +118,7 @@ export const uploadAppRootJson = async ( const url = `${graphConfig.graphMeEndpoint}/me/drive/special/approot:/${fileName}:/content` - return fetch(url, options) + return graphFetch(url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } @@ -114,6 +126,7 @@ export const uploadAppRootJson = async ( export const search = async ( accessToken: string, searchQuery: string, + priority?: GraphRequestPriority, ): Promise => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -127,7 +140,7 @@ export const search = async ( const url = `${graphConfig.graphMeEndpoint}/me/drive/root/search(q='${searchQuery}')` - return fetch(url, options) + return graphFetch(url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) } @@ -136,6 +149,7 @@ export const getDelta = async ( accessToken: string, id?: string, deltaLink?: string, + priority?: GraphRequestPriority, ): Promise => { const headers = new Headers() const bearer = `Bearer ${accessToken}` @@ -149,7 +163,8 @@ export const getDelta = async ( const queryParams = { $top: '2147483647', - $select: 'id,name,parentReference,folder,cTag,deleted,size,lastModifiedDateTime,audio' + // 目前 /delta 接口不会返回 audio 字段 + $select: 'id,name,parentReference,folder,cTag,deleted,size,lastModifiedDateTime' } const param = new URLSearchParams(queryParams) @@ -158,7 +173,7 @@ export const getDelta = async ( ? `${graphConfig.graphMeEndpoint}/me/drive/items/${id}/delta?${param.toString()}` : `${graphConfig.graphMeEndpoint}/me/drive/root/delta?${param.toString()}` - return fetch(deltaLink || url, options) + return graphFetch(deltaLink || url, options, { priority }) .then(response => response.json()) .catch(error => console.log(error)) -} \ No newline at end of file +} diff --git a/src/graph/rateLimiter.ts b/src/graph/rateLimiter.ts new file mode 100644 index 0000000..76554ba --- /dev/null +++ b/src/graph/rateLimiter.ts @@ -0,0 +1,301 @@ +import pLimit from 'p-limit' + +export type RequestPriority = 'high' | 'low' +export type GraphRequestPriority = RequestPriority + +export interface GraphFetchOptions { + priority?: GraphRequestPriority + maxRetries?: number +} + +export interface RateLimitedFetchOptions { + priority?: RequestPriority + maxRetries?: number + scope?: string + addPriorityHeader?: boolean + retryNetworkErrors?: boolean + globalBackoff?: boolean +} + +type ResolvedRateLimitedFetchOptions = Required + +interface RateLimitContext { + input: RequestInfo | URL + method?: string + priority: RequestPriority + retry: number + maxRetries: number + scope: string + globalBackoff: boolean +} + +const DEFAULT_MAX_RETRIES = 2 +const THROTTLED_STATUS_CODES = new Set([429, 503]) + +// Microsoft Graph recommends exponential backoff when Retry-After isn't returned: +// https://learn.microsoft.com/en-us/graph/throttling +// The Microsoft Graph SDK retry delay defaults to 3 seconds and allows up to 180 seconds: +// https://learn.microsoft.com/en-us/dotnet/api/microsoft.graph.retryhandleroption.delay +const FALLBACK_RETRY_BASE_DELAY_MS = 3_000 +const FALLBACK_RETRY_MAX_DELAY_MS = 180_000 +const LOW_PRIORITY_CONCURRENCY = 2 + +let globalBackoffUntil = 0 +let highPriorityBackoffUntil = 0 +let lowPriorityBackoffUntil = 0 +const lowPriorityLimit = pLimit(LOW_PRIORITY_CONCURRENCY) + +export const parseRetryAfterMs = (retryAfter: string | null): number | undefined => { + if (!retryAfter) return undefined + + const seconds = Number(retryAfter) + if (Number.isFinite(seconds)) { + return Math.max(0, seconds * 1000) + } + + const date = Date.parse(retryAfter) + if (Number.isNaN(date)) return undefined + + return Math.max(0, date - Date.now()) +} + +export const parseRateLimitResetMs = (rateLimitReset: string | null): number | undefined => { + if (!rateLimitReset) return undefined + + const seconds = Number(rateLimitReset) + if (Number.isFinite(seconds)) { + return Math.max(0, seconds * 1000) + } + + const date = Date.parse(rateLimitReset) + if (Number.isNaN(date)) return undefined + + return Math.max(0, date - Date.now()) +} + +export const getGraphRetryDelayMs = (headers: Headers): number | undefined => { + const retryAfterMs = parseRetryAfterMs(headers.get('Retry-After')) + const rateLimitResetMs = parseRateLimitResetMs(headers.get('RateLimit-Reset')) + const delays = [retryAfterMs, rateLimitResetMs].filter((delay): delay is number => delay !== undefined) + + if (delays.length === 0) return undefined + + return Math.max(...delays) +} + +const getFallbackRetryDelayMs = (retry: number) => + Math.min(FALLBACK_RETRY_BASE_DELAY_MS * (2 ** retry), FALLBACK_RETRY_MAX_DELAY_MS) + +const getRequestUrl = (input: RequestInfo | URL) => { + if (typeof input === 'string') return input + if (input instanceof URL) return input.toString() + + return input.url +} + +const logRateLimit = ( + message: string, + context: RateLimitContext, + headers: Headers, + status: number, + delayMs: number, +) => { + console.warn(`[${context.scope}] ${message}`, { + status, + method: context.method ?? 'GET', + url: getRequestUrl(context.input), + priority: context.priority, + retry: context.retry, + maxRetries: context.maxRetries, + delayMs, + retryAfter: headers.get('Retry-After'), + rateLimitReset: headers.get('RateLimit-Reset'), + rateLimitRemaining: headers.get('RateLimit-Remaining'), + rateLimitLimit: headers.get('RateLimit-Limit'), + throttleLimitPercentage: headers.get('x-ms-throttle-limit-percentage'), + }) +} + +const sleep = (ms: number, signal?: AbortSignal): Promise => { + if (ms <= 0) return Promise.resolve() + + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason ?? new DOMException('Aborted', 'AbortError')) + return + } + + const timeout = globalThis.setTimeout(() => { + signal?.removeEventListener('abort', handleAbort) + resolve() + }, ms) + + const handleAbort = () => { + globalThis.clearTimeout(timeout) + reject(signal?.reason ?? new DOMException('Aborted', 'AbortError')) + } + + signal?.addEventListener('abort', handleAbort, { once: true }) + }) +} + +const waitForBackoff = async (priority: RequestPriority, signal?: AbortSignal) => { + const now = Date.now() + const backoffUntil = priority === 'low' + ? Math.max(globalBackoffUntil, lowPriorityBackoffUntil) + : Math.max(globalBackoffUntil, highPriorityBackoffUntil) + + await sleep(backoffUntil - now, signal) +} + +const updateBackoffFromHeaders = (headers: Headers, status: number, context: RateLimitContext) => { + const now = Date.now() + const retryDelayMs = getGraphRetryDelayMs(headers) + const rateLimitRemainingHeader = headers.get('RateLimit-Remaining') + const rateLimitLimitHeader = headers.get('RateLimit-Limit') + const rateLimitRemaining = rateLimitRemainingHeader === null ? undefined : Number(rateLimitRemainingHeader) + const rateLimitLimit = rateLimitLimitHeader === null ? undefined : Number(rateLimitLimitHeader) + const isGlobalBackoffTriggered = THROTTLED_STATUS_CODES.has(status) || rateLimitRemaining === 0 + let hasLoggedLowPriorityBackoff = false + + if (isGlobalBackoffTriggered) { + const delayMs = retryDelayMs ?? getFallbackRetryDelayMs(context.retry) + if (context.globalBackoff) { + globalBackoffUntil = Math.max(globalBackoffUntil, now + delayMs) + } else if (context.priority === 'low') { + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) + } else { + highPriorityBackoffUntil = Math.max(highPriorityBackoffUntil, now + delayMs) + } + logRateLimit('Rate limit triggered; backing off requests.', context, headers, status, delayMs) + } + + // Microsoft Graph files/lists resources (drive, driveItem, etc.) use SharePoint limits: + // https://learn.microsoft.com/en-us/graph/throttling-limits#files-and-lists-service-limits + // RateLimit headers are emitted near quota pressure and include RateLimit-Reset; + // https://learn.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online#ratelimit-headers---preview + if ( + rateLimitRemaining !== undefined + && rateLimitLimit !== undefined + && Number.isFinite(rateLimitRemaining) + && Number.isFinite(rateLimitLimit) + && rateLimitLimit > 0 + && rateLimitRemaining / rateLimitLimit < 0.2 + ) { + const delayMs = retryDelayMs ?? FALLBACK_RETRY_BASE_DELAY_MS + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) + if (!isGlobalBackoffTriggered) { + logRateLimit('Rate limit quota is low; backing off low-priority requests.', context, headers, status, delayMs) + hasLoggedLowPriorityBackoff = true + } + } + + const throttleLimitPercentage = Number(headers.get('x-ms-throttle-limit-percentage')) + if (Number.isFinite(throttleLimitPercentage) && throttleLimitPercentage >= 0.8) { + const delayMs = retryDelayMs ?? FALLBACK_RETRY_BASE_DELAY_MS + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, now + delayMs) + if (!isGlobalBackoffTriggered && !hasLoggedLowPriorityBackoff) { + logRateLimit('Throttle limit is high; backing off low-priority requests.', context, headers, status, delayMs) + } + } +} + +const runRateLimitedFetch = async ( + input: RequestInfo | URL, + init: RequestInit | undefined, + options: ResolvedRateLimitedFetchOptions, +): Promise => { + const signal = init?.signal ?? undefined + + for (let retry = 0; retry <= options.maxRetries; retry += 1) { + await waitForBackoff(options.priority, signal) + + const requestInit = options.addPriorityHeader + ? (() => { + const headers = new Headers(init?.headers) + // https://learn.microsoft.com/en-us/graph/throttling-limits + headers.set('x-ms-throttle-priority', options.priority === 'low' ? 'Low' : 'High') + return { ...init, headers } + })() + : init + + try { + const response = await fetch(input, requestInit) + updateBackoffFromHeaders(response.headers, response.status, { + input, + method: init?.method, + priority: options.priority, + retry, + maxRetries: options.maxRetries, + scope: options.scope, + globalBackoff: options.globalBackoff, + }) + + if (!THROTTLED_STATUS_CODES.has(response.status) || retry >= options.maxRetries) return response + } catch (error) { + if (signal?.aborted || !options.retryNetworkErrors || retry >= options.maxRetries) throw error + const delayMs = getFallbackRetryDelayMs(retry) + if (options.globalBackoff) { + globalBackoffUntil = Math.max(globalBackoffUntil, Date.now() + delayMs) + } else if (options.priority === 'low') { + lowPriorityBackoffUntil = Math.max(lowPriorityBackoffUntil, Date.now() + delayMs) + } else { + highPriorityBackoffUntil = Math.max(highPriorityBackoffUntil, Date.now() + delayMs) + } + console.warn(`[${options.scope}] Request failed; backing off requests.`, { + url: getRequestUrl(input), + priority: options.priority, + retry, + maxRetries: options.maxRetries, + delayMs, + error, + }) + } + } + + throw new Error('Unexpected rate-limited fetch retry state.') +} + +const enqueueLowPriorityFetch = ( + input: RequestInfo | URL, + init: RequestInit | undefined, + options: ResolvedRateLimitedFetchOptions, +): Promise => lowPriorityLimit(() => runRateLimitedFetch(input, init, options)) + +export const rateLimitedFetch = ( + input: RequestInfo | URL, + init?: RequestInit, + options: RateLimitedFetchOptions = {}, +): Promise => { + const resolvedOptions: ResolvedRateLimitedFetchOptions = { + priority: options.priority ?? 'high', + maxRetries: options.maxRetries ?? DEFAULT_MAX_RETRIES, + scope: options.scope ?? 'External', + addPriorityHeader: options.addPriorityHeader ?? false, + retryNetworkErrors: options.retryNetworkErrors ?? false, + globalBackoff: options.globalBackoff ?? false, + } + + if (resolvedOptions.priority === 'low') { + return enqueueLowPriorityFetch(input, init, resolvedOptions) + } + + return runRateLimitedFetch(input, init, resolvedOptions) +} + +export const graphFetch = ( + input: RequestInfo | URL, + init?: RequestInit, + { + priority = 'high', + maxRetries = DEFAULT_MAX_RETRIES, + }: GraphFetchOptions = {}, +): Promise => { + return rateLimitedFetch(input, init, { + priority, + maxRetries, + scope: 'Graph', + addPriorityHeader: true, + globalBackoff: true, + }) +} diff --git a/src/hooks/graph/useGraph.ts b/src/hooks/graph/useGraph.ts index b46b275..4ce14b5 100644 --- a/src/hooks/graph/useGraph.ts +++ b/src/hooks/graph/useGraph.ts @@ -1,6 +1,7 @@ import { getAppRootFiles, getFile, getFiles, search, uploadAppRootJson, getDelta } from '@/graph/graph' import { loginRequest } from '@/graph/authConfig' import { AccountInfo, IPublicClientApplication } from '@azure/msal-browser' +import { GraphRequestPriority } from '@/graph/rateLimiter' const useGraph = ( instance: IPublicClientApplication, @@ -20,54 +21,60 @@ const useGraph = ( return tokenResponse.accessToken } - const getFilesData = async (id: string, path?: string[]) => { + const getFilesData = async (id: string, path?: string[], priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - let response = await getFiles(accessToken, id, path,) + let response = await getFiles(accessToken, id, path, undefined, priority) const remoteItems = [...response.value] while (response['@odata.nextLink']) { - response = await getFiles(accessToken, id, path, response['@odata.nextLink']) + response = await getFiles(accessToken, id, path, response['@odata.nextLink'], priority) remoteItems.push(...response.value) } return { value: remoteItems } } - const getFileData = async (id: string, path?: string[], signal?: AbortSignal) => { + const getFileData = async ( + id: string, + path?: string[], + signal?: AbortSignal, + priority?: GraphRequestPriority, + includeThumbnails = true, + ) => { const accessToken = await getAccessToken() - const response = await getFile(accessToken, id, path, signal) + const response = await getFile(accessToken, id, path, signal, priority, includeThumbnails) return response } - const getAppRootFilesData = async () => { + const getAppRootFilesData = async (priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - const response = await getAppRootFiles(accessToken) + const response = await getAppRootFiles(accessToken, priority) return response } - const uploadAppRootJsonData = async (fileName: string, fileContent: BodyInit) => { + const uploadAppRootJsonData = async (fileName: string, fileContent: BodyInit, priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - const response = await uploadAppRootJson(accessToken, fileName, fileContent) + const response = await uploadAppRootJson(accessToken, fileName, fileContent, priority) return response } - const getSearchData = async (searchQuery: string) => { + const getSearchData = async (searchQuery: string, priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - const response = await search(accessToken, searchQuery) + const response = await search(accessToken, searchQuery, priority) return response } - const getDeltaData = async (id: string, deltaLink?: string) => { + const getDeltaData = async (id: string, deltaLink?: string, priority?: GraphRequestPriority) => { const accessToken = await getAccessToken() - let response = await getDelta(accessToken, id, deltaLink) + let response = await getDelta(accessToken, id, deltaLink, priority) const remoteItems = [...response.value] while (response['@odata.nextLink']) { - response = await getDelta(accessToken, id, response['@odata.nextLink']) + response = await getDelta(accessToken, id, response['@odata.nextLink'], priority) remoteItems.push(...response.value) } @@ -81,7 +88,6 @@ const useGraph = ( } } - return { getFilesData, getFileData, diff --git a/src/hooks/graph/useMetadataSync.ts b/src/hooks/graph/useMetadataSync.ts new file mode 100644 index 0000000..4a25b9b --- /dev/null +++ b/src/hooks/graph/useMetadataSync.ts @@ -0,0 +1,179 @@ +import { useEffect, useRef } from 'react' +import useUser from './useUser' +import useDb from '../useDb' +import useGraph from './useGraph' +import { useMsal } from '@azure/msal-react' +import useFileNodeSyncStore from '@/store/useFileNodeSyncStore' +import useMetadataSyncStore from '@/store/useMetadataSyncStore' +import { graphAudioToMetadata } from '@/utils/graphAudioMetadata' +import type { FileNode } from '@/types/file' +import { getRangeMetadata } from '@/utils/rangeMetadata' +import { RangeRequestError } from '@/utils/rangeReader' +import pLimit from 'p-limit' + +const MAX_ATTEMPTS = 3 +const RETRY_BASE_DELAY_MS = 1_000 +const metadataTaskLimit = pLimit(2) + +const getAbortReason = (signal: AbortSignal) => + signal.reason ?? new DOMException('Aborted', 'AbortError') + +const throwIfAborted = (signal: AbortSignal) => { + if (signal.aborted) throw getAbortReason(signal) +} + +const wait = (ms: number, signal: AbortSignal) => new Promise((resolve, reject) => { + throwIfAborted(signal) + + const timeout = window.setTimeout(() => { + signal.removeEventListener('abort', handleAbort) + resolve() + }, ms) + + const handleAbort = () => { + window.clearTimeout(timeout) + reject(getAbortReason(signal)) + } + + signal.addEventListener('abort', handleAbort, { once: true }) +}) + +const useMetadataSync = () => { + const { instance } = useMsal() + const { account } = useUser() + const db = useDb(account) + const { getFileData } = useGraph(instance, account) + const fileSyncStatus = useFileNodeSyncStore.use.status() + const start = useMetadataSyncStore.use.start() + const finish = useMetadataSyncStore.use.finish() + const running = useRef(null) + + useEffect(() => { + if (!db || !account || fileSyncStatus !== 'success' || running.current) return + + const controller = new AbortController() + const { signal } = controller + const runToken = Symbol('metadata-sync') + let started = false + running.current = runToken + + const isCurrentRun = () => running.current === runToken && !signal.aborted + + const fetchWithRetry = async (node: FileNode) => { + let lastError: unknown + + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) { + throwIfAborted(signal) + try { + return await getFileData(node.id, undefined, signal, 'low', false) + } catch (error) { + throwIfAborted(signal) + lastError = error + if (attempt < MAX_ATTEMPTS - 1) { + await wait(RETRY_BASE_DELAY_MS * (2 ** attempt), signal) + } + } + } + + throw lastError + } + + const run = async () => { + try { + const nodes = await db.nodes + .where('metadataState') + .anyOf(['pending', 'failed']) + .and(node => node.type === 'audio') + .toArray() + + if (nodes.length === 0 || !isCurrentRun()) return + + start() + started = true + + await Promise.allSettled(nodes.map(node => metadataTaskLimit(async () => { + if (!isCurrentRun()) return + + try { + const remoteItem = await fetchWithRetry(node) + if (!isCurrentRun()) return + + const graphTitle = typeof remoteItem.audio?.title === 'string' && remoteItem.audio.title.trim() + let metadata + + if (graphTitle) { + metadata = graphAudioToMetadata(node, remoteItem.audio) + } else if (remoteItem['@microsoft.graph.downloadUrl']) { + try { + // Range requests must target the short-lived download URL. + // https://learn.microsoft.com/en-us/graph/api/driveitem-get-content?view=graph-rest-1.0#partial-range-downloads + metadata = await getRangeMetadata(node, remoteItem['@microsoft.graph.downloadUrl'], signal) + } catch (error) { + if (!isCurrentRun()) return + if (error instanceof RangeRequestError) throw error + console.warn(`Falling back to filename metadata for ${node.name}:`, error) + metadata = graphAudioToMetadata(node) + } + } else { + throw new Error('No download URL returned for metadata extraction.') + } + + if (!isCurrentRun()) return + await db.transaction('rw', db.metadata, db.nodes, async transaction => { + const handleAbort = () => transaction.abort() + signal.addEventListener('abort', handleAbort, { once: true }) + try { + throwIfAborted(signal) + await db.metadata.put(metadata) + throwIfAborted(signal) + await db.nodes.update(node.id, { metadataState: 'completed' }) + } finally { + signal.removeEventListener('abort', handleAbort) + } + }) + } catch (error) { + if (!isCurrentRun()) return + console.error(`Failed to get metadata for ${node.name}:`, error) + try { + await db.transaction('rw', db.nodes, async transaction => { + const handleAbort = () => transaction.abort() + signal.addEventListener('abort', handleAbort, { once: true }) + try { + throwIfAborted(signal) + await db.nodes.update(node.id, { metadataState: 'failed' }) + } finally { + signal.removeEventListener('abort', handleAbort) + } + }) + } catch (dbError) { + if (isCurrentRun()) { + console.error(`Failed to update metadataState to failed for ${node.name}:`, dbError) + } + } + } + }))) + } catch (error) { + if (isCurrentRun()) console.error('Failed to start metadata sync:', error) + } finally { + if (running.current === runToken) { + running.current = null + if (started) finish() + } + } + } + + void run() + + return () => { + controller.abort() + if (running.current === runToken) { + running.current = null + if (started) finish() + } + } + // Graph helpers close over the current account and are recreated on render. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [account, db, fileSyncStatus, finish, start]) +} + +export default useMetadataSync diff --git a/src/hooks/player/useMetaData.ts b/src/hooks/player/useMetaData.ts index 8d9ba70..629977e 100644 --- a/src/hooks/player/useMetaData.ts +++ b/src/hooks/player/useMetaData.ts @@ -69,12 +69,15 @@ const useMetaData = (url: string) => { (async () => { if (currentTrack && currentTrack.track.id && isAudio(currentTrack.track.name) && db && url) { const localMetaData = await db.metadata.get(currentTrack.track.id) - if (!localMetaData) { + if (!localMetaData || localMetaData.source !== 'stream') { console.log('Start get net metadata: ', currentTrack.track) const result = await getNetMetaData(currentTrack.track, url) if (result) { - await db.metadata.put(result.metaData) - await db.pictures.bulkPut(result.pictureData) + await db.transaction('rw', db.metadata, db.pictures, db.nodes, async () => { + await db.metadata.put(result.metaData) + await db.pictures.bulkPut(result.pictureData) + await db.nodes.update(currentTrack.track.id, { metadataState: 'completed' }) + }) updateMetadataUpdate() } } @@ -87,4 +90,4 @@ const useMetaData = (url: string) => { } -export default useMetaData \ No newline at end of file +export default useMetaData diff --git a/src/locales/en/messages.po b/src/locales/en/messages.po index 53f15ed..3aba085 100644 --- a/src/locales/en/messages.po +++ b/src/locales/en/messages.po @@ -14,7 +14,7 @@ msgstr "" "Plural-Forms: \n" #. placeholder {0}: songs.length -#: src/pages/Library/AlbumDetail.tsx:136 +#: src/pages/Library/AlbumDetail.tsx:165 #: src/pages/Library/FolderDetail.tsx:90 msgid "{0} songs" msgstr "{0} songs" @@ -48,11 +48,11 @@ msgstr "Add to play queue" msgid "Add to playlist" msgstr "Add to playlist" -#: src/pages/Library/Library.tsx:61 +#: src/pages/Library/Library.tsx:64 msgid "Album" msgstr "Album" -#: src/pages/Library/Library.tsx:70 +#: src/pages/Library/Library.tsx:73 msgid "Artist" msgstr "Artist" @@ -127,7 +127,7 @@ msgstr "Descending" msgid "Enter new title" msgstr "Enter new title" -#: src/pages/Library/Library.tsx:109 +#: src/pages/Library/Library.tsx:115 msgid "Error" msgstr "Error" @@ -135,7 +135,7 @@ msgstr "Error" msgid "Files" msgstr "Files" -#: src/pages/Library/Library.tsx:88 +#: src/pages/Library/Library.tsx:91 msgid "Folder" msgstr "Folder" @@ -160,7 +160,7 @@ msgstr "HD thumbnails" msgid "History" msgstr "History" -#: src/pages/Library/Library.tsx:107 +#: src/pages/Library/Library.tsx:113 msgid "Idle" msgstr "Idle" @@ -177,7 +177,7 @@ msgstr "Library" msgid "Library folder" msgstr "Library folder" -#: src/pages/Library/Library.tsx:120 +#: src/pages/Library/Library.tsx:126 msgid "Library folder not set" msgstr "Library folder not set" @@ -326,7 +326,7 @@ msgstr "Sign out" msgid "Size" msgstr "Size" -#: src/pages/Library/Library.tsx:79 +#: src/pages/Library/Library.tsx:82 msgid "Song" msgstr "Song" @@ -339,7 +339,7 @@ msgstr "Switch fullscreen" msgid "Switch theme" msgstr "Switch theme" -#: src/pages/Library/Library.tsx:108 +#: src/pages/Library/Library.tsx:114 msgid "Syncing" msgstr "Syncing" diff --git a/src/locales/zh-CN/messages.po b/src/locales/zh-CN/messages.po index cfc7160..865e373 100644 --- a/src/locales/zh-CN/messages.po +++ b/src/locales/zh-CN/messages.po @@ -14,7 +14,7 @@ msgstr "" "Plural-Forms: \n" #. placeholder {0}: songs.length -#: src/pages/Library/AlbumDetail.tsx:136 +#: src/pages/Library/AlbumDetail.tsx:165 #: src/pages/Library/FolderDetail.tsx:90 msgid "{0} songs" msgstr "{0} 首歌曲" @@ -48,11 +48,11 @@ msgstr "添加到播放队列" msgid "Add to playlist" msgstr "添加到播放列表" -#: src/pages/Library/Library.tsx:61 +#: src/pages/Library/Library.tsx:64 msgid "Album" msgstr "专辑" -#: src/pages/Library/Library.tsx:70 +#: src/pages/Library/Library.tsx:73 msgid "Artist" msgstr "艺术家" @@ -127,7 +127,7 @@ msgstr "倒序" msgid "Enter new title" msgstr "输入新标题" -#: src/pages/Library/Library.tsx:109 +#: src/pages/Library/Library.tsx:115 msgid "Error" msgstr "错误" @@ -135,7 +135,7 @@ msgstr "错误" msgid "Files" msgstr "文件" -#: src/pages/Library/Library.tsx:88 +#: src/pages/Library/Library.tsx:91 msgid "Folder" msgstr "文件夹" @@ -160,7 +160,7 @@ msgstr "高清缩略图" msgid "History" msgstr "历史" -#: src/pages/Library/Library.tsx:107 +#: src/pages/Library/Library.tsx:113 msgid "Idle" msgstr "空闲" @@ -177,7 +177,7 @@ msgstr "媒体库" msgid "Library folder" msgstr "媒体库文件夹" -#: src/pages/Library/Library.tsx:120 +#: src/pages/Library/Library.tsx:126 msgid "Library folder not set" msgstr "未设置媒体库文件夹" @@ -326,7 +326,7 @@ msgstr "注销" msgid "Size" msgstr "大小" -#: src/pages/Library/Library.tsx:79 +#: src/pages/Library/Library.tsx:82 msgid "Song" msgstr "歌曲" @@ -339,7 +339,7 @@ msgstr "切换全屏" msgid "Switch theme" msgstr "切换主题" -#: src/pages/Library/Library.tsx:108 +#: src/pages/Library/Library.tsx:114 msgid "Syncing" msgstr "同步中" diff --git a/src/pages/Library/Library.tsx b/src/pages/Library/Library.tsx index 6afe608..a99b1f5 100644 --- a/src/pages/Library/Library.tsx +++ b/src/pages/Library/Library.tsx @@ -11,6 +11,7 @@ import Loading from '../Loading' import { useMemo } from 'react' import { Link, Outlet, useLocation } from 'react-router-dom' import useFileNodeSyncStore from '@/store/useFileNodeSyncStore' +import useMetadataSyncStore from '@/store/useMetadataSyncStore' import SetLibraryFolderDialog from '@/components/Dialog/SetLibraryFolderDialog' const Library = () => { @@ -25,6 +26,7 @@ const Library = () => { const status = useFileNodeSyncStore.use.status() const error = useFileNodeSyncStore.use.error() + const metadataStatus = useMetadataSyncStore.use.status() const isLoadingSettings = settings === undefined @@ -51,9 +53,10 @@ const Library = () => { status === 'success' ? - + } @@ -92,6 +95,9 @@ const Library = () => { value="/library/folders" /> + + {metadataStatus === 'fetching' && } + @@ -123,4 +129,4 @@ const Library = () => { ) } -export default Library \ No newline at end of file +export default Library diff --git a/src/store/useMetadataSyncStore.ts b/src/store/useMetadataSyncStore.ts new file mode 100644 index 0000000..ef92e5c --- /dev/null +++ b/src/store/useMetadataSyncStore.ts @@ -0,0 +1,20 @@ +import { create } from 'zustand' +import createSelectors from './createSelectors' + +type MetadataSyncStatus = 'idle' | 'fetching' + +interface MetadataSyncStore { + status: MetadataSyncStatus + start: () => void + finish: () => void +} + +const useMetadataSyncStoreBase = create((set) => ({ + status: 'idle', + start: () => set({ status: 'fetching' }), + finish: () => set({ status: 'idle' }), +})) + +const useMetadataSyncStore = createSelectors(useMetadataSyncStoreBase) + +export default useMetadataSyncStore diff --git a/src/types/MetaData.ts b/src/types/MetaData.ts index e6d2020..ff05bfa 100644 --- a/src/types/MetaData.ts +++ b/src/types/MetaData.ts @@ -11,9 +11,10 @@ export interface PicutreData { export interface MetaData { id: string + source?: 'graph' | 'range' | 'filename' | 'stream' common: Omit & { albumartists?: string[] picture?: Picture[] } format: IAudioMetadata['format'] -} \ No newline at end of file +} diff --git a/src/types/file.ts b/src/types/file.ts index df05696..232ce41 100644 --- a/src/types/file.ts +++ b/src/types/file.ts @@ -9,6 +9,27 @@ export interface DeltaResponse { value: RemoteItem[] } +// The audio facet is populated by Microsoft Graph only for OneDrive Personal. +// https://learn.microsoft.com/en-us/graph/api/resources/audio?view=graph-rest-1.0 +export interface AudioDetails { + album?: string + albumArtist?: string + artist?: string + bitrate?: number + composers?: string + copyright?: string + disc?: number + discCount?: number + duration?: number + genre?: string + hasDrm?: boolean + isVariableBitrate?: boolean + title?: string + track?: number + trackCount?: number + year?: number +} + export interface FileDetails { mimeType: string hashes?: { @@ -74,6 +95,7 @@ export interface RemoteItem { file?: FileDetails folder?: FolderDetails + audio?: AudioDetails parentReference: ParentReference @@ -117,4 +139,4 @@ export interface Track { path: string[] size: number cTag?: string -} \ No newline at end of file +} diff --git a/src/utils/getNetMetaData.ts b/src/utils/getNetMetaData.ts index 292cfba..3171564 100644 --- a/src/utils/getNetMetaData.ts +++ b/src/utils/getNetMetaData.ts @@ -1,6 +1,7 @@ import { FileNode, Track } from '@/types/file' import { MetaData, Picture, PicutreData } from '@/types/metaData' import { IPicture, parseWebStream } from 'music-metadata' +import { rateLimitedFetch } from '@/graph/rateLimiter' export const compressImage = async (image: IPicture): Promise => { const blob = new Blob([image.data as unknown as ArrayBuffer], { type: image.format }) @@ -67,7 +68,10 @@ const getSha256 = async (data: Uint8Array): Promise => { const getNetMetaData = async (file: FileNode | Track, url: string): Promise<{ metaData: MetaData, pictureData: PicutreData[] } | null> => { try { - const response = await fetch(url) + const response = await rateLimitedFetch(url, undefined, { + scope: 'Content', + retryNetworkErrors: true, + }) if (response.body === null) { return null @@ -122,6 +126,7 @@ const getNetMetaData = async (file: FileNode | Track, url: string): Promise<{ me const metaData: MetaData = { id: file.id, + source: 'stream', common: { ...metadata.common, title: metadata.common.title.trim(), diff --git a/src/utils/graphAudioMetadata.ts b/src/utils/graphAudioMetadata.ts new file mode 100644 index 0000000..81fc56f --- /dev/null +++ b/src/utils/graphAudioMetadata.ts @@ -0,0 +1,47 @@ +import { AudioDetails, FileNode } from '@/types/file' +import { MetaData } from '@/types/metaData' + +const filenameTitle = (name: string) => name.replace(/\.[^.]+$/, '').trim() || name +const clean = (value?: unknown) => typeof value === 'string' ? value.trim() || undefined : undefined +const positive = (value?: number) => typeof value === 'number' && Number.isFinite(value) && value > 0 + ? value + : undefined + +export const graphAudioToMetadata = (file: FileNode, audio?: AudioDetails): MetaData => { + const graphTitle = clean(audio?.title) + const artist = clean(audio?.artist) + const albumartist = clean(audio?.albumArtist) + const album = clean(audio?.album) + const genre = clean(audio?.genre) + const composer = clean(audio?.composers) + const track = positive(audio?.track) + const trackCount = positive(audio?.trackCount) + const disk = positive(audio?.disc) + const diskCount = positive(audio?.discCount) + const duration = positive(audio?.duration) + const bitrate = positive(audio?.bitrate) + + return { + id: file.id, + source: graphTitle ? 'graph' : 'filename', + format: { + duration: duration === undefined ? undefined : duration / 1000, + bitrate: bitrate === undefined ? undefined : bitrate * 1000, + }, + native: {}, + quality: { warnings: [] }, + common: { + title: graphTitle ?? filenameTitle(file.name), + artist, + artists: artist ? [artist] : undefined, + albumartist, + album, + composer: composer ? [composer] : undefined, + genre: genre ? [genre] : undefined, + year: positive(audio?.year), + track: { no: track, of: trackCount }, + disk: { no: disk, of: diskCount }, + copyright: clean(audio?.copyright) ? [clean(audio?.copyright)!] : undefined, + }, + } +} diff --git a/src/utils/rangeMetadata.test.ts b/src/utils/rangeMetadata.test.ts new file mode 100644 index 0000000..cddc727 --- /dev/null +++ b/src/utils/rangeMetadata.test.ts @@ -0,0 +1,221 @@ +import { afterEach, describe, it } from 'node:test' +import assert from 'node:assert/strict' +import { getRangeMetadata } from './rangeMetadata.ts' +import { HttpRangeReader, RangeBudgetExceededError, RangeNotSupportedError, RangeRequestError } from './rangeReader.ts' +import type { FileNode } from '@/types/file' + +const originalFetch = globalThis.fetch +const concat = (...parts: Uint8Array[]) => { + const result = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)) + let offset = 0 + for (const part of parts) { + result.set(part, offset) + offset += part.length + } + return result +} + +const ascii = (value: string) => new TextEncoder().encode(value) +const u32be = (value: number) => new Uint8Array([(value >>> 24) & 255, (value >>> 16) & 255, (value >>> 8) & 255, value & 255]) +const u32le = (value: number) => new Uint8Array([value & 255, (value >>> 8) & 255, (value >>> 16) & 255, (value >>> 24) & 255]) +const syncSafe = (value: number) => new Uint8Array([(value >>> 21) & 127, (value >>> 14) & 127, (value >>> 7) & 127, value & 127]) +const frame = (id: string, payload: Uint8Array) => concat(ascii(id), u32be(payload.length), new Uint8Array(2), payload) +const flacBlock = (type: number, payload: Uint8Array, last = false) => concat( + new Uint8Array([(last ? 0x80 : 0) | type, (payload.length >>> 16) & 255, (payload.length >>> 8) & 255, payload.length & 255]), + payload, +) +const wavChunk = (id: string, payload: Uint8Array) => concat( + ascii(id), + u32le(payload.length), + payload, + payload.length % 2 === 0 ? new Uint8Array() : new Uint8Array(1), +) +const wavFile = (...chunks: Uint8Array[]) => { + const body = concat(ascii('WAVE'), ...chunks) + return concat(ascii('RIFF'), u32le(body.length), body) +} + +const mockRanges = (file: Uint8Array, status = 206) => { + const requested: Array<[number, number]> = [] + globalThis.fetch = async (_input: string | URL | Request, init?: RequestInit) => { + const range = new Headers(init?.headers).get('Range')! + const match = range.match(/^bytes=(\d+)-(\d+)$/)! + const start = Number(match[1]) + const end = Number(match[2]) + requested.push([start, end]) + if (status !== 206) return new Response(file, { status }) + return new Response(file.slice(start, end + 1), { + status: 206, + headers: { 'Content-Range': `bytes ${start}-${end}/${file.length}` }, + }) + } + return requested +} + +afterEach(() => { globalThis.fetch = originalFetch }) + +describe('HttpRangeReader', () => { + it('rejects a server that ignores Range', async () => { + mockRanges(new Uint8Array(32), 200) + await assert.rejects(() => new HttpRangeReader('test').read(0, 4), RangeNotSupportedError) + }) + + it('enforces the cumulative byte budget before requesting excess bytes', async () => { + const file = new Uint8Array(1024 * 1024 + 1) + const requested = mockRanges(file) + const reader = new HttpRangeReader('test') + await reader.read(0, 1024 * 1024) + await assert.rejects(() => reader.read(1024 * 1024, 1), RangeBudgetExceededError) + assert.equal(requested.length, 16) + }) + + it('accepts and caches an aligned partial response covering the requested range', async () => { + const file = new Uint8Array(1024).map((_, index) => index % 256) + let requests = 0 + globalThis.fetch = async () => { + requests += 1 + return new Response(file.slice(0, 512), { + status: 206, + headers: { + 'Content-Range': `bytes 0-511/${file.length}`, + 'Content-Length': '512', + }, + }) + } + const reader = new HttpRangeReader('test') + + assert.deepEqual(await reader.read(100, 4), file.slice(100, 104)) + assert.deepEqual(await reader.read(200, 4), file.slice(200, 204)) + assert.equal(requests, 1) + assert.equal(reader.bytesTransferred, 512) + }) + + it('accepts a 206 response when CORS hides Content-Range', async () => { + const file = new Uint8Array([10, 20, 30, 40]) + globalThis.fetch = async () => new Response(file, { + status: 206, + headers: { 'Content-Length': String(file.length) }, + }) + + const reader = new HttpRangeReader('test') + assert.deepEqual(await reader.read(0, 4), file) + assert.equal(reader.bytesTransferred, 4) + }) + + it('does not issue a request when already aborted', async () => { + let requests = 0 + globalThis.fetch = async () => { + requests += 1 + return new Response(new Uint8Array(4), { status: 206 }) + } + const controller = new AbortController() + controller.abort() + + await assert.rejects(() => new HttpRangeReader('test', undefined, controller.signal).read(0, 4), { + name: 'AbortError', + }) + assert.equal(requests, 0) + }) + + it('passes cancellation to an in-flight range request', async () => { + let notifyStarted!: () => void + const started = new Promise(resolve => { notifyStarted = resolve }) + globalThis.fetch = async (_input, init) => new Promise((_resolve, reject) => { + notifyStarted() + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }) + }) + const controller = new AbortController() + const reading = new HttpRangeReader('test', undefined, controller.signal).read(0, 4) + + await started + controller.abort() + + await assert.rejects(() => reading, RangeRequestError) + }) +}) + +describe('getRangeMetadata', () => { + it('extracts ID3 text frames without requesting APIC payload bytes', async () => { + const title = frame('TIT2', concat(new Uint8Array([3]), ascii('Range title'))) + const picture = frame('APIC', new Uint8Array(200_000).fill(7)) + const artist = frame('TPE1', concat(new Uint8Array([3]), ascii('Range artist'))) + const body = concat(title, picture, artist) + const file = concat(ascii('ID3'), new Uint8Array([3, 0, 0]), syncSafe(body.length), body, new Uint8Array(100)) + const requested = mockRanges(file) + const pictureStart = 10 + title.length + 10 + const pictureEnd = pictureStart + 200_000 - 1 + const node = { id: '1', name: 'song.mp3', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'Range title') + assert.equal(metadata.common.artist, 'Range artist') + assert.equal(metadata.source, 'range') + assert.equal(requested.some(([start, end]) => start <= pictureEnd && end >= pictureStart), false) + }) + + it('skips a FLAC PICTURE block and reads following Vorbis comments', async () => { + const picture = new Uint8Array(200_000).fill(9) + const vendor = ascii('test') + const comment = ascii('TITLE=FLAC title') + const comments = concat(u32le(vendor.length), vendor, u32le(1), u32le(comment.length), comment) + const file = concat(ascii('fLaC'), flacBlock(6, picture), flacBlock(4, comments, true)) + const requested = mockRanges(file) + const pictureStart = 8 + const pictureEnd = pictureStart + picture.length - 1 + const node = { id: '2', name: 'song.flac', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'FLAC title') + assert.equal(requested.some(([start, end]) => start <= pictureEnd && end >= pictureStart), false) + }) + + it('extracts a title from a valid WAV LIST INFO chunk', async () => { + const title = ascii('WAV title\0') + const info = concat(ascii('INFO'), wavChunk('INAM', title)) + const file = wavFile(wavChunk('LIST', info)) + mockRanges(file) + const node = { id: '3', name: 'song.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'WAV title') + assert.equal(metadata.source, 'range') + }) + + it('does not read beyond a truncated outer WAV chunk', async () => { + const file = concat(ascii('RIFF'), u32le(16), ascii('WAVE'), ascii('LIST'), u32le(64)) + const requested = mockRanges(file) + const node = { id: '4', name: 'truncated.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'truncated') + assert.equal(requested.every(([, end]) => end < file.length), true) + }) + + it('does not read an INFO item payload beyond its LIST chunk', async () => { + const invalidItem = concat(ascii('INAM'), u32le(100)) + const file = wavFile(wavChunk('LIST', concat(ascii('INFO'), invalidItem))) + const requested = mockRanges(file) + const node = { id: '5', name: 'invalid-info.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'invalid-info') + assert.equal(requested.every(([, end]) => end < file.length), true) + }) + + it('bounds an embedded ID3 tag to its WAV chunk', async () => { + const oversizedId3 = concat(ascii('ID3'), new Uint8Array([3, 0, 0]), syncSafe(100)) + const file = wavFile(wavChunk('ID3 ', oversizedId3)) + const requested = mockRanges(file) + const node = { id: '6', name: 'invalid-id3.wav', size: file.length } as FileNode + + const metadata = await getRangeMetadata(node, 'test') + + assert.equal(metadata.common.title, 'invalid-id3') + assert.equal(requested.every(([, end]) => end < file.length), true) + }) +}) diff --git a/src/utils/rangeMetadata.ts b/src/utils/rangeMetadata.ts new file mode 100644 index 0000000..bcad74e --- /dev/null +++ b/src/utils/rangeMetadata.ts @@ -0,0 +1,378 @@ +import type { FileNode } from '@/types/file' +import type { MetaData } from '@/types/metaData' +import { HttpRangeReader } from './rangeReader.ts' + +type TextTags = Partial<{ + title: string + artist: string + albumartist: string + album: string + composer: string + genre: string + year: number + track: number + trackTotal: number + disk: number + diskTotal: number + copyright: string +}> + +const decoder = new TextDecoder() +const ascii = (bytes: Uint8Array) => String.fromCharCode(...bytes) +const u32be = (b: Uint8Array, o = 0) => new DataView(b.buffer, b.byteOffset, b.byteLength).getUint32(o) +const u32le = (b: Uint8Array, o = 0) => new DataView(b.buffer, b.byteOffset, b.byteLength).getUint32(o, true) +const syncSafe = (b: Uint8Array, o = 0) => ((b[o] & 0x7f) << 21) | ((b[o + 1] & 0x7f) << 14) | ((b[o + 2] & 0x7f) << 7) | (b[o + 3] & 0x7f) +const trim = (value: string) => value.replace(/\0/g, '').trim() +const numberPair = (value: string) => { + const [no, of] = value.split('/').map(item => Number.parseInt(item, 10)) + return { no: Number.isFinite(no) ? no : undefined, of: Number.isFinite(of) ? of : undefined } +} + +const decodeText = (bytes: Uint8Array, encoding = 3) => { + if (bytes.length === 0) return '' + if (encoding === 0) return trim(new TextDecoder('windows-1252').decode(bytes)) + if (encoding === 3) return trim(decoder.decode(bytes)) + if (encoding === 2) return trim(new TextDecoder('utf-16be').decode(bytes)) + if (bytes[0] === 0xff && bytes[1] === 0xfe) return trim(new TextDecoder('utf-16le').decode(bytes.subarray(2))) + if (bytes[0] === 0xfe && bytes[1] === 0xff) return trim(new TextDecoder('utf-16be').decode(bytes.subarray(2))) + return trim(new TextDecoder('utf-16le').decode(bytes)) +} + +const applyTag = (tags: TextTags, key: string, value: string) => { + const normalized = key.toUpperCase() + if (!value) return + if (['TIT2', 'TT2', 'TITLE', '©NAM', 'INAM'].includes(normalized)) tags.title = value + else if (['TPE1', 'TP1', 'ARTIST', '©ART', 'IART'].includes(normalized)) tags.artist = value + else if (['TPE2', 'TP2', 'ALBUMARTIST', 'ALBUM ARTIST', 'AART'].includes(normalized)) tags.albumartist = value + else if (['TALB', 'TAL', 'ALBUM', '©ALB', 'IPRD'].includes(normalized)) tags.album = value + else if (['TCOM', 'TCM', 'COMPOSER', '©WRT', 'IWRI'].includes(normalized)) tags.composer = value + else if (['TCON', 'TCO', 'GENRE', '©GEN', 'IGNR'].includes(normalized)) tags.genre = value + else if (['TYER', 'TYE', 'TDRC', 'DATE', 'YEAR', '©DAY', 'ICRD'].includes(normalized)) tags.year = Number.parseInt(value, 10) || undefined + else if (['TRCK', 'TRK', 'TRACKNUMBER'].includes(normalized)) Object.assign(tags, { track: numberPair(value).no, trackTotal: numberPair(value).of }) + else if (normalized === 'TRACKTOTAL' || normalized === 'TOTALTRACKS') tags.trackTotal = Number.parseInt(value, 10) || undefined + else if (['TPOS', 'TPA', 'DISCNUMBER'].includes(normalized)) Object.assign(tags, { disk: numberPair(value).no, diskTotal: numberPair(value).of }) + else if (normalized === 'DISCTOTAL' || normalized === 'TOTALDISCS') tags.diskTotal = Number.parseInt(value, 10) || undefined + else if (['TCOP', 'TCR', 'COPYRIGHT', 'ICOP'].includes(normalized)) tags.copyright = value +} + +const parseId3 = async (reader: HttpRangeReader, base = 0, maxEnd = Number.POSITIVE_INFINITY): Promise => { + const tags: TextTags = {} + if (base + 10 > maxEnd) return tags + const header = await reader.read(base, 10) + if (ascii(header.subarray(0, 3)) !== 'ID3') return tags + const version = header[3] + const end = base + 10 + syncSafe(header, 6) + if (end > maxEnd) return tags + let offset = base + 10 + const frameHeaderLength = version === 2 ? 6 : 10 + const idLength = version === 2 ? 3 : 4 + const idPattern = version === 2 ? /^[A-Z0-9]{3}$/ : /^[A-Z0-9]{4}$/ + + while (offset + frameHeaderLength <= end) { + const frameHeader = await reader.read(offset, frameHeaderLength) + const id = ascii(frameHeader.subarray(0, idLength)) + if (!idPattern.test(id)) break + const size = version === 2 + ? (frameHeader[3] << 16) | (frameHeader[4] << 8) | frameHeader[5] + : version === 4 ? syncSafe(frameHeader, 4) : u32be(frameHeader, 4) + offset += frameHeaderLength + if (size <= 0 || offset + size > end) break + + if (id.startsWith('T') && id !== 'TXXX') { + const data = await reader.read(offset, size) + applyTag(tags, id, decodeText(data.subarray(1), data[0])) + } else if (id === 'COMM' && size > 4) { + const data = await reader.read(offset, size) + const encoding = data[0] + const separator = encoding === 0 || encoding === 3 ? 1 : 2 + let pos = 4 + while (pos + separator <= data.length && data.subarray(pos, pos + separator).some(byte => byte !== 0)) pos += separator + const value = decodeText(data.subarray(pos + separator), encoding) + if (value && !tags.title) applyTag(tags, 'COMMENT', value) + } + offset += size + } + + return tags +} + +const parseId3v1 = async (reader: HttpRangeReader, fileSize: number): Promise => { + if (fileSize < 128) return {} + const data = await reader.read(fileSize - 128, 128) + if (ascii(data.subarray(0, 3)) !== 'TAG') return {} + const tags: TextTags = {} + applyTag(tags, 'TITLE', decodeText(data.subarray(3, 33), 0)) + applyTag(tags, 'ARTIST', decodeText(data.subarray(33, 63), 0)) + applyTag(tags, 'ALBUM', decodeText(data.subarray(63, 93), 0)) + applyTag(tags, 'YEAR', decodeText(data.subarray(93, 97), 0)) + if (data[125] === 0 && data[126] > 0) tags.track = data[126] + return tags +} + +const parseFlacComments = async (reader: HttpRangeReader, start: number, length: number): Promise => { + const tags: TextTags = {} + const end = start + length + let offset = start + const vendorLength = u32le(await reader.read(offset, 4)); offset += 4 + vendorLength + const count = u32le(await reader.read(offset, 4)); offset += 4 + + for (let index = 0; index < count && offset + 4 <= end; index += 1) { + const commentLength = u32le(await reader.read(offset, 4)); offset += 4 + if (offset + commentLength > end) throw new Error('Invalid FLAC comment length.') + const prefixLength = Math.min(commentLength, 128) + const prefix = await reader.read(offset, prefixLength) + const prefixText = decoder.decode(prefix) + const separator = prefixText.indexOf('=') + const key = separator < 0 ? '' : prefixText.slice(0, separator).toUpperCase() + + if (!['METADATA_BLOCK_PICTURE', 'COVERART', 'COVERARTMIME'].includes(key)) { + const rest = commentLength > prefixLength + ? await reader.read(offset + prefixLength, commentLength - prefixLength) + : new Uint8Array() + const value = decoder.decode(concatBytes(prefix, rest)) + const valueSeparator = value.indexOf('=') + if (valueSeparator > 0) applyTag(tags, value.slice(0, valueSeparator), trim(value.slice(valueSeparator + 1))) + } + offset += commentLength + } + return tags +} + +const parseFlac = async (reader: HttpRangeReader): Promise => { + let offset = 4 + while (true) { + const header = await reader.read(offset, 4) + const last = (header[0] & 0x80) !== 0 + const type = header[0] & 0x7f + const length = (header[1] << 16) | (header[2] << 8) | header[3] + offset += 4 + if (type === 4) return parseFlacComments(reader, offset, length) + offset += length + if (last) return {} + } +} + +type Atom = { name: string, start: number, dataStart: number, end: number } +const readAtom = async (reader: HttpRangeReader, start: number, parentEnd: number): Promise => { + if (start + 8 > parentEnd) return null + const header = await reader.read(start, 8) + let size = u32be(header) + const name = ascii(header.subarray(4, 8)) + let headerSize = 8 + if (size === 1) { + const large = await reader.read(start + 8, 8) + size = Number(new DataView(large.buffer, large.byteOffset).getBigUint64(0)) + headerSize = 16 + } else if (size === 0) size = parentEnd - start + if (size < headerSize || start + size > parentEnd) return null + return { name, start, dataStart: start + headerSize, end: start + size } +} + +const parseMp4 = async (reader: HttpRangeReader, fileSize: number): Promise => { + const tags: TextTags = {} + const containers = new Set(['moov', 'udta', 'meta', 'ilst']) + const walk = async (start: number, end: number, inIlst = false) => { + let offset = start + while (offset + 8 <= end) { + const atom = await readAtom(reader, offset, end) + if (!atom) break + if (containers.has(atom.name)) { + await walk(atom.dataStart + (atom.name === 'meta' ? 4 : 0), atom.end, atom.name === 'ilst' || inIlst) + } else if (inIlst && atom.name !== 'covr') { + let childOffset = atom.dataStart + while (childOffset + 16 <= atom.end) { + const child = await readAtom(reader, childOffset, atom.end) + if (!child) break + if (child.name === 'data' && child.end - child.dataStart >= 8) { + const payload = await reader.read(child.dataStart + 8, child.end - child.dataStart - 8) + if (atom.name === 'trkn' && payload.length >= 6) { + tags.track = (payload[2] << 8) | payload[3] + tags.trackTotal = (payload[4] << 8) | payload[5] + } else if (atom.name === 'disk' && payload.length >= 6) { + tags.disk = (payload[2] << 8) | payload[3] + tags.diskTotal = (payload[4] << 8) | payload[5] + } else applyTag(tags, atom.name, trim(decoder.decode(payload))) + } + childOffset = child.end + } + } + offset = atom.end + } + } + await walk(0, fileSize) + return tags +} + +const parseWav = async (reader: HttpRangeReader, fileSize: number): Promise => { + const tags: TextTags = {} + let offset = 12 + while (offset + 8 <= fileSize) { + const header = await reader.read(offset, 8) + const id = ascii(header.subarray(0, 4)) + const size = u32le(header, 4) + const dataStart = offset + 8 + const dataEnd = dataStart + size + if (dataEnd > fileSize) break + if (id === 'ID3 ' || id === 'id3 ') Object.assign(tags, await parseId3(reader, dataStart, dataEnd)) + if (id === 'LIST' && size >= 4 && ascii(await reader.read(dataStart, 4)) === 'INFO') { + let itemOffset = dataStart + 4 + while (itemOffset + 8 <= dataEnd) { + const itemHeader = await reader.read(itemOffset, 8) + const itemSize = u32le(itemHeader, 4) + const itemDataStart = itemOffset + 8 + const itemDataEnd = itemDataStart + itemSize + if (itemDataEnd > dataEnd) break + applyTag(tags, ascii(itemHeader.subarray(0, 4)), trim(decoder.decode(await reader.read(itemDataStart, itemSize)))) + itemOffset = itemDataEnd + (itemSize % 2) + } + } + offset = dataEnd + (size % 2) + } + return tags +} + +type OggSegment = { offset: number, length: number } + +class OggPacketReader { + private position = 0 + private readonly reader: HttpRangeReader + private readonly segments: OggSegment[] + + constructor(reader: HttpRangeReader, segments: OggSegment[]) { + this.reader = reader + this.segments = segments + } + + async read(length: number): Promise { + const output = new Uint8Array(length) + let outputOffset = 0 + let logicalOffset = 0 + + for (const segment of this.segments) { + const segmentEnd = logicalOffset + segment.length + if (this.position < segmentEnd && outputOffset < length) { + const withinSegment = Math.max(0, this.position - logicalOffset) + const take = Math.min(segment.length - withinSegment, length - outputOffset) + output.set(await this.reader.read(segment.offset + withinSegment, take), outputOffset) + this.position += take + outputOffset += take + } + logicalOffset = segmentEnd + if (outputOffset === length) return output + } + throw new Error('Unexpected end of Ogg packet.') + } + + skip(length: number) { + this.position += length + } +} + +const parseOgg = async (reader: HttpRangeReader, fileSize: number): Promise => { + const packets: OggSegment[][] = [] + let currentPacket: OggSegment[] = [] + let offset = 0 + + while (offset + 27 <= fileSize && packets.length < 2) { + const header = await reader.read(offset, 27) + if (ascii(header.subarray(0, 4)) !== 'OggS') break + const segmentCount = header[26] + const lacing = await reader.read(offset + 27, segmentCount) + let payloadOffset = offset + 27 + segmentCount + + for (const segmentLength of lacing) { + currentPacket.push({ offset: payloadOffset, length: segmentLength }) + payloadOffset += segmentLength + if (segmentLength < 255) { + packets.push(currentPacket) + currentPacket = [] + } + } + offset = payloadOffset + } + + if (packets.length < 2) return {} + const packet = new OggPacketReader(reader, packets[1]) + const signature = await packet.read(8) + if (ascii(signature.subarray(0, 7)) === '\x03vorbis') packet.skip(-1) + else if (ascii(signature) !== 'OpusTags') return {} + + const vendorLength = u32le(await packet.read(4)) + packet.skip(vendorLength) + const count = u32le(await packet.read(4)) + const tags: TextTags = {} + + for (let index = 0; index < count; index += 1) { + const length = u32le(await packet.read(4)) + const prefixLength = Math.min(length, 128) + const prefix = await packet.read(prefixLength) + const prefixText = decoder.decode(prefix) + const separator = prefixText.indexOf('=') + const key = separator < 0 ? '' : prefixText.slice(0, separator).toUpperCase() + const remaining = length - prefixLength + + if (['METADATA_BLOCK_PICTURE', 'COVERART', 'COVERARTMIME'].includes(key)) { + packet.skip(remaining) + continue + } + + const rest = remaining > 0 ? await packet.read(remaining) : new Uint8Array() + const value = decoder.decode(concatBytes(prefix, rest)) + const valueSeparator = value.indexOf('=') + if (valueSeparator > 0) applyTag(tags, value.slice(0, valueSeparator), trim(value.slice(valueSeparator + 1))) + } + + return tags +} + +const filenameTitle = (name: string) => name.replace(/\.[^.]+$/, '').trim() || name + +export const getRangeMetadata = async (file: FileNode, url: string, signal?: AbortSignal): Promise => { + const reader = new HttpRangeReader(url, undefined, signal) + const signature = await reader.read(0, Math.min(4, file.size)) + let tags: TextTags = {} + + if (ascii(signature.subarray(0, 3)) === 'ID3') tags = await parseId3(reader, 0, file.size) + else if (ascii(signature.subarray(0, 4)) === 'fLaC') tags = await parseFlac(reader) + else if (ascii(signature.subarray(0, 4)) === 'OggS') tags = await parseOgg(reader, file.size) + else if (file.size >= 12) { + const extendedSignature = concatBytes(signature, await reader.read(4, 8)) + if (ascii(extendedSignature.subarray(4, 8)) === 'ftyp') tags = await parseMp4(reader, file.size) + else if (ascii(signature) === 'RIFF' && ascii(extendedSignature.subarray(8, 12)) === 'WAVE') tags = await parseWav(reader, file.size) + } + + if (/\.mp3$/i.test(file.name) && !tags.title) { + tags = { ...await parseId3v1(reader, file.size), ...tags } + } + + return { + id: file.id, + source: tags.title ? 'range' : 'filename', + format: {}, + native: {}, + quality: { warnings: [] }, + common: { + title: tags.title ?? filenameTitle(file.name), + artist: tags.artist, + artists: tags.artist ? [tags.artist] : undefined, + albumartist: tags.albumartist, + album: tags.album, + composer: tags.composer ? [tags.composer] : undefined, + genre: tags.genre ? [tags.genre] : undefined, + year: tags.year, + track: { no: tags.track, of: tags.trackTotal }, + disk: { no: tags.disk, of: tags.diskTotal }, + copyright: tags.copyright ? [tags.copyright] : undefined, + }, + } +} + +const concatBytes = (...parts: Uint8Array[]) => { + const output = new Uint8Array(parts.reduce((total, part) => total + part.length, 0)) + let offset = 0 + for (const part of parts) { + output.set(part, offset) + offset += part.length + } + return output +} diff --git a/src/utils/rangeReader.ts b/src/utils/rangeReader.ts new file mode 100644 index 0000000..c69bd88 --- /dev/null +++ b/src/utils/rangeReader.ts @@ -0,0 +1,128 @@ +import { rateLimitedFetch } from '../graph/rateLimiter.ts' + +export class RangeBudgetExceededError extends Error { + constructor() { + super('Range download budget exceeded.') + } +} + +export class RangeNotSupportedError extends Error { + constructor(details?: string) { + super(`Server did not return a valid partial response${details ? `: ${details}` : '.'}`) + } +} + +export class RangeRequestError extends Error { + constructor(cause: unknown) { + super('Range request failed.', { cause }) + } +} + +const MAX_REQUEST_SIZE = 64 * 1024 + +export class HttpRangeReader { + private transferred = 0 + private readonly cache: Array<{ start: number, end: number, bytes: Uint8Array }> = [] + private readonly url: string + private readonly budget: number + private readonly signal?: AbortSignal + + constructor(url: string, budget = 1024 * 1024, signal?: AbortSignal) { + this.url = url + this.budget = budget + this.signal = signal + } + + get bytesTransferred() { + return this.transferred + } + + async read(offset: number, length: number): Promise { + this.throwIfAborted() + + if (offset < 0 || length < 0 || !Number.isSafeInteger(offset) || !Number.isSafeInteger(length)) { + throw new RangeError('Invalid byte range.') + } + + const result = new Uint8Array(length) + let written = 0 + + while (written < length) { + this.throwIfAborted() + const chunkLength = Math.min(MAX_REQUEST_SIZE, length - written) + const start = offset + written + const end = start + chunkLength - 1 + const cached = this.cache.find(entry => entry.start <= start && entry.end >= end) + if (cached) { + result.set(cached.bytes.subarray(start - cached.start, end - cached.start + 1), written) + written += chunkLength + continue + } + + if (this.transferred + chunkLength > this.budget) throw new RangeBudgetExceededError() + const response = await this.fetchRange(start, end) + + if (response.status !== 206) { + await response.body?.cancel() + throw new RangeNotSupportedError(`status=${response.status}, requested=${start}-${end}`) + } + + const contentRange = response.headers.get('Content-Range') + const match = contentRange?.match(/^bytes (\d+)-(\d+)\/(\d+|\*)$/) + let responseStart = match ? Number(match[1]) : start + let responseEnd = match ? Number(match[2]) : end + if (match && (responseStart > start || responseEnd < end)) { + await response.body?.cancel() + throw new RangeNotSupportedError(`Content-Range=${contentRange ?? 'missing'}, requested=${start}-${end}`) + } + + const contentLengthHeader = response.headers.get('Content-Length') + const contentLength = contentLengthHeader === null ? undefined : Number(contentLengthHeader) + if (contentLength !== undefined && Number.isFinite(contentLength) && this.transferred + contentLength > this.budget) { + await response.body?.cancel() + throw new RangeBudgetExceededError() + } + + const bytes = new Uint8Array(await response.arrayBuffer()) + if (!match) { + if (bytes.byteLength < chunkLength) { + throw new RangeNotSupportedError(`received=${bytes.byteLength}, requested=${start}-${end}, Content-Range is not exposed`) + } + // OneDrive may hide Content-Range from browser code while returning a + // valid 206 response. Treat the body as starting at the requested byte. + responseStart = start + responseEnd = start + bytes.byteLength - 1 + } + const expectedLength = responseEnd - responseStart + 1 + if (bytes.byteLength !== expectedLength) { + throw new RangeNotSupportedError(`received=${bytes.byteLength}, Content-Range=${contentRange}`) + } + this.transferred += bytes.byteLength + if (this.transferred > this.budget) throw new RangeBudgetExceededError() + this.cache.push({ start: responseStart, end: responseEnd, bytes }) + + result.set(bytes.subarray(start - responseStart, end - responseStart + 1), written) + written += chunkLength + } + + return result + } + + private async fetchRange(start: number, end: number) { + try { + return await rateLimitedFetch( + this.url, + { headers: { Range: `bytes=${start}-${end}` }, signal: this.signal }, + { priority: 'low', scope: 'Content', retryNetworkErrors: true }, + ) + } catch (error) { + throw new RangeRequestError(error) + } + } + + private throwIfAborted() { + if (this.signal?.aborted) { + throw this.signal.reason ?? new DOMException('Aborted', 'AbortError') + } + } +} diff --git a/src/utils/remote.ts b/src/utils/remote.ts index 245b2d6..25b4980 100644 --- a/src/utils/remote.ts +++ b/src/utils/remote.ts @@ -1,5 +1,6 @@ import { RemoteItem, FileNode } from '@/types/file' import checkFileType from './checkFileType' +import { rateLimitedFetch } from '@/graph/rateLimiter' /** * 根据 url 解析 json @@ -8,7 +9,10 @@ import checkFileType from './checkFileType' */ export const fetchJson = async (url: string) => { try { - const response = await fetch(url) + const response = await rateLimitedFetch(url, undefined, { + scope: 'External', + retryNetworkErrors: true, + }) const json = response.json() return json } catch (error) { @@ -60,4 +64,4 @@ export const sizeConv = (fileSize: number) => { : ((fileSize / 1024 / 1024) < 1024) ? `${(fileSize / 1024 / 1024).toFixed(2)} MB` : `${(fileSize / 1024 / 1024 / 1024).toFixed(2)} GB` -} \ No newline at end of file +} diff --git a/tsconfig.json b/tsconfig.json index 456caf1..4537626 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,7 @@ "noUnusedParameters": true, "noFallthroughCasesInSwitch": true, "allowSyntheticDefaultImports": true, + "allowImportingTsExtensions": true, "esModuleInterop": true, "noImplicitAny": true, "allowJs": false, @@ -35,4 +36,4 @@ "**/*.ts", "**/*.tsx", ] -} \ No newline at end of file +}