From 740c6c1200a99ffd85fd01048f167c2ca27e06bd Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sat, 4 Jul 2026 08:42:43 +0000 Subject: [PATCH] Optimize cloning lists utilizing BatchWriteCommand for citation insertions Co-authored-by: aicoder2009 <127642633+aicoder2009@users.noreply.github.com> --- .jules/bolt.md | 4 ++ src/app/api/share/[code]/clone/route.ts | 34 ++-------- src/lib/db/dynamodb.ts | 84 ++++++++++++++++++++++++- src/lib/db/index.ts | 1 + src/lib/db/local-store.ts | 37 +++++++++++ 5 files changed, 129 insertions(+), 31 deletions(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index 43fafef..b27b65b 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -5,3 +5,7 @@ ## 2024-05-18 - Concurrent Fetching with Owner IDs in Share Links **Learning:** In the share system, endpoints processing a share code typically suffer from waterfall latency by first loading target entity details (like a project) and then querying its associated elements (like project lists) using the entity's owner ID (`userId`). However, the `shareLink` object already contains the `userId` field (representing the owner's ID). **Action:** Always leverage the existing owner's ID within `shareLink` to bypass sequential dependencies and fetch parent entities (like projects) concurrently with their child elements (like user lists) using `Promise.all`. + +## 2024-05-19 - Batch Operations for DynamoDB N+1 Insert Latency +**Learning:** During iterative data duplication tasks like cloning projects or lists, mapping over array insertions using `Promise.all` triggers a severe N+1 database request bottleneck. +**Action:** Always prefer `BatchWriteCommand` over iteration. By chunking sequential array insertions into groups of 25 items (DynamoDB's batch limit) via custom functions like `batchAddCitations`, we drastically reduce I/O overhead, optimizing write-speed and scaling capacity safely. diff --git a/src/app/api/share/[code]/clone/route.ts b/src/app/api/share/[code]/clone/route.ts index 6e32b1d..2f7a97a 100644 --- a/src/app/api/share/[code]/clone/route.ts +++ b/src/app/api/share/[code]/clone/route.ts @@ -11,6 +11,7 @@ import { createList, createProject, addCitation, + batchAddCitations, reorderCitations, } from "@/lib/db"; import { parseShareSegment } from "@/lib/share-utils"; @@ -111,21 +112,7 @@ export async function POST( const newList = await createList(userId, originalList.name, undefined, originalList.description); - const newCitations = await Promise.all( - citations.map((c) => - addCitation( - newList.id, - c.fields, - c.style, - c.formattedText, - c.formattedHtml, - c.tags, - c.notes, - c.quotes, - c.readingStatus - ) - ) - ); + const newCitations = await batchAddCitations(newList.id, citations); if (citations.some((c) => c.sortOrder !== undefined)) { await reorderCitations(newList.id, newCitations.map((c) => c.id)); @@ -155,21 +142,8 @@ export async function POST( originalLists.map(async (list) => { const citations = await getListCitations(list.id); const newList = await createList(userId, list.name, newProject.id, list.description); - const newCitations = await Promise.all( - citations.map((c) => - addCitation( - newList.id, - c.fields, - c.style, - c.formattedText, - c.formattedHtml, - c.tags, - c.notes, - c.quotes, - c.readingStatus - ) - ) - ); + const newCitations = await batchAddCitations(newList.id, citations); + if (citations.some((c) => c.sortOrder !== undefined)) { await reorderCitations(newList.id, newCitations.map((c) => c.id)); } diff --git a/src/lib/db/dynamodb.ts b/src/lib/db/dynamodb.ts index 1ef4c90..9f064c8 100644 --- a/src/lib/db/dynamodb.ts +++ b/src/lib/db/dynamodb.ts @@ -288,7 +288,6 @@ export async function deleteList(userId: string, listId: string): Promise if (citations.length > 0) { // DynamoDB allows a maximum of 25 items per BatchWriteItem request const BATCH_SIZE = 25; - const batches = []; for (let i = 0; i < citations.length; i += BATCH_SIZE) { const batch = citations.slice(i, i + BATCH_SIZE); @@ -479,6 +478,89 @@ export async function getProjectLists(userId: string, projectId: string): Promis // ============ CITATIONS ============ +export async function batchAddCitations( + listId: string, + citationsData: { + fields: CitationFields; + style: CitationStyle; + formattedText: string; + formattedHtml: string; + tags?: string[]; + notes?: string; + quotes?: CitationQuote[]; + readingStatus?: ReadingStatus; + }[] +): Promise { + const now = new Date().toISOString(); + + const citations: Citation[] = citationsData.map((data) => ({ + id: generateId(), + listId, + fields: data.fields, + style: data.style, + formattedText: data.formattedText, + formattedHtml: data.formattedHtml, + tags: data.tags, + notes: data.notes, + quotes: data.quotes, + readingStatus: data.readingStatus, + createdAt: now, + updatedAt: now, + })); + + if (citations.length > 0) { + // DynamoDB allows a maximum of 25 items per BatchWriteItem request + const BATCH_SIZE = 25; + + for (let i = 0; i < citations.length; i += BATCH_SIZE) { + const batch = citations.slice(i, i + BATCH_SIZE); + + let requestItems = { + [TABLE_NAME]: batch.map((citation) => ({ + PutRequest: { + Item: { + PK: keys.list(listId), + SK: keys.citation(citation.id), + GSI1PK: keys.citation(citation.id), + GSI1SK: keys.list(listId), + ...citation, + entityType: "CITATION", + }, + }, + })), + }; + + let retries = 0; + const MAX_RETRIES = 3; + + while (Object.keys(requestItems).length > 0 && retries <= MAX_RETRIES) { + const response = await docClient.send( + new BatchWriteCommand({ + RequestItems: requestItems, + }) + ); + + if ( + response.UnprocessedItems && + Object.keys(response.UnprocessedItems).length > 0 + ) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + requestItems = response.UnprocessedItems as any; + retries++; + if (retries <= MAX_RETRIES) { + // Exponential backoff + await new Promise((resolve) => setTimeout(resolve, Math.pow(2, retries) * 100)); + } + } else { + break; + } + } + } + } + + return citations; +} + export async function addCitation( listId: string, fields: CitationFields, diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index c334587..0c8ce80 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -26,6 +26,7 @@ export const {getProjectLists} = db; // ============ CITATIONS ============ export const {addCitation} = db; +export const {batchAddCitations} = db; export const {getCitation} = db; export const {getListCitations} = db; export const {updateCitation} = db; diff --git a/src/lib/db/local-store.ts b/src/lib/db/local-store.ts index c95c5dd..1856ec7 100644 --- a/src/lib/db/local-store.ts +++ b/src/lib/db/local-store.ts @@ -223,6 +223,43 @@ export async function getProjectLists(userId: string, projectId: string): Promis // ============ CITATIONS ============ +export async function batchAddCitations( + listId: string, + citationsData: { + fields: CitationFields; + style: CitationStyle; + formattedText: string; + formattedHtml: string; + tags?: string[]; + notes?: string; + quotes?: CitationQuote[]; + readingStatus?: ReadingStatus; + }[] +): Promise { + const now = new Date().toISOString(); + + const citations: Citation[] = citationsData.map((data) => ({ + id: generateId(), + listId, + fields: data.fields, + style: data.style, + formattedText: data.formattedText, + formattedHtml: data.formattedHtml, + tags: data.tags, + notes: data.notes, + quotes: data.quotes, + readingStatus: data.readingStatus, + createdAt: now, + updatedAt: now, + })); + + citations.forEach(citation => { + store.citations.set(`${listId}:${citation.id}`, citation); + }); + + return citations; +} + export async function addCitation( listId: string, fields: CitationFields,