From ef3ba163ddc54ec003c610020cb3885e9c11443d Mon Sep 17 00:00:00 2001 From: Frederic Cabassut Date: Fri, 6 Mar 2026 17:12:39 +0100 Subject: [PATCH] fix: resolve N+1 queries, add indexes, and improve DB operations - Batch-fetch tags for notes in a single query instead of per-note loops - Add composite indexes on notes, note_tags, and sync_log tables - Wrap sync push in a transaction for atomicity - Replace O(n) cleanExpiredSessions with single DELETE query - Use SQL WHERE in getChangesSince instead of loading all notes - Extract shared syncNoteTags utility to eliminate duplication - Add root +error.svelte boundary matching app theme Co-Authored-By: Claude Opus 4.6 --- src/lib/server/auth.ts | 11 +-- src/lib/server/db/index.ts | 6 ++ src/lib/server/db/schema.ts | 84 ++++++++++++--------- src/lib/server/tags.ts | 46 ++++++++++++ src/lib/sync/server.ts | 105 +++++++++++++++------------ src/routes/+error.svelte | 25 +++++++ src/routes/api/notes/+server.ts | 48 ++++-------- src/routes/api/notes/[id]/+server.ts | 47 ++++-------- src/routes/api/search/+server.ts | 51 ++++++------- 9 files changed, 237 insertions(+), 186 deletions(-) create mode 100644 src/lib/server/tags.ts create mode 100644 src/routes/+error.svelte diff --git a/src/lib/server/auth.ts b/src/lib/server/auth.ts index 12ba76e..bc5d444 100644 --- a/src/lib/server/auth.ts +++ b/src/lib/server/auth.ts @@ -1,6 +1,6 @@ import { db } from './db/index.js'; import { users, sessions } from './db/schema.js'; -import { eq } from 'drizzle-orm'; +import { eq, lt } from 'drizzle-orm'; import * as argon2 from 'argon2'; import { randomBytes } from 'crypto'; @@ -61,12 +61,5 @@ export async function deleteSession(token: string): Promise { } export async function cleanExpiredSessions(): Promise { - const now = new Date(); - // Delete all expired sessions by checking each one - const allSessions = await db.select().from(sessions); - for (const session of allSessions) { - if (session.expiresAt < now) { - await db.delete(sessions).where(eq(sessions.id, session.id)); - } - } + await db.delete(sessions).where(lt(sessions.expiresAt, new Date())); } diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index fc967b6..78177ac 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -76,6 +76,12 @@ sqlite.exec(` timestamp INTEGER NOT NULL, client_id TEXT NOT NULL ); + + CREATE INDEX IF NOT EXISTS notes_trashed_archived_idx ON notes(trashed, archived); + CREATE INDEX IF NOT EXISTS notes_updated_at_idx ON notes(updated_at); + CREATE INDEX IF NOT EXISTS note_tags_note_id_idx ON note_tags(note_id); + CREATE INDEX IF NOT EXISTS note_tags_tag_id_idx ON note_tags(tag_id); + CREATE INDEX IF NOT EXISTS sync_log_timestamp_idx ON sync_log(timestamp); `); export const db = drizzle(sqlite, { schema }); diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index 54cae96..78abb53 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer, uniqueIndex } from 'drizzle-orm/sqlite-core'; +import { sqliteTable, text, integer, uniqueIndex, index } from 'drizzle-orm/sqlite-core'; export const users = sqliteTable('users', { id: integer('id').primaryKey({ autoIncrement: true }), @@ -14,21 +14,28 @@ export const sessions = sqliteTable('sessions', { expiresAt: integer('expires_at', { mode: 'timestamp' }).notNull() }); -export const notes = sqliteTable('notes', { - id: text('id').primaryKey(), - title: text('title').default('').notNull(), - content: text('content').default('').notNull(), - color: text('color').default('default').notNull(), - pinned: integer('pinned', { mode: 'boolean' }).default(false).notNull(), - archived: integer('archived', { mode: 'boolean' }).default(false).notNull(), - trashed: integer('trashed', { mode: 'boolean' }).default(false).notNull(), - trashedAt: integer('trashed_at', { mode: 'timestamp' }), - checklistMode: integer('checklist_mode', { mode: 'boolean' }).default(false).notNull(), - sortOrder: integer('sort_order').default(0).notNull(), - createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), - updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(), - version: integer('version').default(1).notNull() -}); +export const notes = sqliteTable( + 'notes', + { + id: text('id').primaryKey(), + title: text('title').default('').notNull(), + content: text('content').default('').notNull(), + color: text('color').default('default').notNull(), + pinned: integer('pinned', { mode: 'boolean' }).default(false).notNull(), + archived: integer('archived', { mode: 'boolean' }).default(false).notNull(), + trashed: integer('trashed', { mode: 'boolean' }).default(false).notNull(), + trashedAt: integer('trashed_at', { mode: 'timestamp' }), + checklistMode: integer('checklist_mode', { mode: 'boolean' }).default(false).notNull(), + sortOrder: integer('sort_order').default(0).notNull(), + createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), + updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(), + version: integer('version').default(1).notNull() + }, + (table) => [ + index('notes_trashed_archived_idx').on(table.trashed, table.archived), + index('notes_updated_at_idx').on(table.updatedAt) + ] +); export const tags = sqliteTable( 'tags', @@ -39,14 +46,21 @@ export const tags = sqliteTable( (table) => [uniqueIndex('tags_name_unique').on(table.name)] ); -export const noteTags = sqliteTable('note_tags', { - noteId: text('note_id') - .references(() => notes.id, { onDelete: 'cascade' }) - .notNull(), - tagId: integer('tag_id') - .references(() => tags.id, { onDelete: 'cascade' }) - .notNull() -}); +export const noteTags = sqliteTable( + 'note_tags', + { + noteId: text('note_id') + .references(() => notes.id, { onDelete: 'cascade' }) + .notNull(), + tagId: integer('tag_id') + .references(() => tags.id, { onDelete: 'cascade' }) + .notNull() + }, + (table) => [ + index('note_tags_note_id_idx').on(table.noteId), + index('note_tags_tag_id_idx').on(table.tagId) + ] +); export const attachments = sqliteTable('attachments', { id: text('id').primaryKey(), @@ -60,12 +74,16 @@ export const attachments = sqliteTable('attachments', { createdAt: integer('created_at', { mode: 'timestamp' }).notNull() }); -export const syncLog = sqliteTable('sync_log', { - id: integer('id').primaryKey({ autoIncrement: true }), - noteId: text('note_id') - .references(() => notes.id) - .notNull(), - operation: text('operation').notNull(), - timestamp: integer('timestamp', { mode: 'timestamp' }).notNull(), - clientId: text('client_id').notNull() -}); +export const syncLog = sqliteTable( + 'sync_log', + { + id: integer('id').primaryKey({ autoIncrement: true }), + noteId: text('note_id') + .references(() => notes.id) + .notNull(), + operation: text('operation').notNull(), + timestamp: integer('timestamp', { mode: 'timestamp' }).notNull(), + clientId: text('client_id').notNull() + }, + (table) => [index('sync_log_timestamp_idx').on(table.timestamp)] +); diff --git a/src/lib/server/tags.ts b/src/lib/server/tags.ts new file mode 100644 index 0000000..f193334 --- /dev/null +++ b/src/lib/server/tags.ts @@ -0,0 +1,46 @@ +import { db } from './db/index.js'; +import { noteTags, tags } from './db/schema.js'; +import { eq, inArray } from 'drizzle-orm'; + +/** + * Batch fetch tags for multiple notes in a single query. + * Returns a Map of noteId → tag names. + */ +export function fetchTagsForNotes(noteIds: string[]): Map { + if (noteIds.length === 0) return new Map(); + + const rows = db + .select({ noteId: noteTags.noteId, name: tags.name }) + .from(noteTags) + .innerJoin(tags, eq(noteTags.tagId, tags.id)) + .where(inArray(noteTags.noteId, noteIds)) + .all(); + + const map = new Map(); + for (const row of rows) { + const existing = map.get(row.noteId); + if (existing) { + existing.push(row.name); + } else { + map.set(row.noteId, [row.name]); + } + } + return map; +} + +/** + * Sync tags for a note: removes old associations, upserts tags, creates new associations. + * Should be called within a transaction for atomicity. + */ +export function syncNoteTags(noteId: string, tagNames: string[]) { + db.delete(noteTags).where(eq(noteTags.noteId, noteId)).run(); + + for (const name of tagNames) { + let tagRow = db.select().from(tags).where(eq(tags.name, name)).get(); + if (!tagRow) { + const result = db.insert(tags).values({ name }).returning().all(); + tagRow = result[0]; + } + db.insert(noteTags).values({ noteId, tagId: tagRow.id }).run(); + } +} diff --git a/src/lib/sync/server.ts b/src/lib/sync/server.ts index ceb41f7..7e7392c 100644 --- a/src/lib/sync/server.ts +++ b/src/lib/sync/server.ts @@ -7,65 +7,74 @@ import type { SyncQueueItem } from './idb.js'; * Process incoming sync changes from client. */ export async function processSyncPush(changes: SyncQueueItem[]): Promise { - for (const change of changes) { - switch (change.operation) { - case 'create': - case 'update': { - if (!change.data) continue; - const existing = await db.select().from(notes).where(eq(notes.id, change.noteId)).get(); + db.transaction((tx) => { + for (const change of changes) { + switch (change.operation) { + case 'create': + case 'update': { + if (!change.data) continue; + const existing = tx + .select() + .from(notes) + .where(eq(notes.id, change.noteId)) + .get(); - if (existing) { - // LWW: only update if incoming timestamp is newer - if (change.timestamp > existing.updatedAt.getTime()) { - await db - .update(notes) - .set({ - ...change.data, + if (existing) { + if (change.timestamp > existing.updatedAt.getTime()) { + tx.update(notes) + .set({ + ...change.data, + updatedAt: new Date(change.timestamp), + version: existing.version + 1 + }) + .where(eq(notes.id, change.noteId)) + .run(); + } + } else if (change.operation === 'create' && change.data) { + tx.insert(notes) + .values({ + id: change.noteId, + title: change.data.title || '', + content: change.data.content || '', + color: change.data.color || 'default', + pinned: change.data.pinned || false, + archived: change.data.archived || false, + trashed: change.data.trashed || false, + checklistMode: change.data.checklistMode || false, + sortOrder: change.data.sortOrder || 0, + createdAt: new Date(change.timestamp), updatedAt: new Date(change.timestamp), - version: existing.version + 1 + version: 1 }) - .where(eq(notes.id, change.noteId)); + .run(); } - } else if (change.operation === 'create' && change.data) { - await db.insert(notes).values({ - id: change.noteId, - title: change.data.title || '', - content: change.data.content || '', - color: change.data.color || 'default', - pinned: change.data.pinned || false, - archived: change.data.archived || false, - trashed: change.data.trashed || false, - checklistMode: change.data.checklistMode || false, - sortOrder: change.data.sortOrder || 0, - createdAt: new Date(change.timestamp), - updatedAt: new Date(change.timestamp), - version: 1 - }); + break; + } + case 'delete': { + tx.delete(notes).where(eq(notes.id, change.noteId)).run(); + break; } - break; } - case 'delete': { - await db.delete(notes).where(eq(notes.id, change.noteId)); - break; - } - } - // Log sync operation - await db.insert(syncLog).values({ - noteId: change.noteId, - operation: change.operation, - timestamp: new Date(change.timestamp), - clientId: 'default' - }); - } + tx.insert(syncLog) + .values({ + noteId: change.noteId, + operation: change.operation, + timestamp: new Date(change.timestamp), + clientId: 'default' + }) + .run(); + } + }); } /** * Get all notes updated since a given timestamp. */ export async function getChangesSince(sinceTimestamp: number) { - const since = new Date(sinceTimestamp); - const allNotes = await db.select().from(notes); - // Filter notes updated after the timestamp - return allNotes.filter((n) => n.updatedAt > since); + return db + .select() + .from(notes) + .where(gt(notes.updatedAt, new Date(sinceTimestamp))) + .all(); } diff --git a/src/routes/+error.svelte b/src/routes/+error.svelte new file mode 100644 index 0000000..e192979 --- /dev/null +++ b/src/routes/+error.svelte @@ -0,0 +1,25 @@ + + + + Error - Crumbs + + +
+
+ 🥨 +

+ {page.status} +

+

+ {page.error?.message ?? 'Something went wrong'} +

+ + Back to notes + +
+
diff --git a/src/routes/api/notes/+server.ts b/src/routes/api/notes/+server.ts index 8d9a68b..b01b0c2 100644 --- a/src/routes/api/notes/+server.ts +++ b/src/routes/api/notes/+server.ts @@ -1,10 +1,11 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types.js'; import { db } from '$lib/server/db/index.js'; -import { notes, noteTags, tags } from '$lib/server/db/schema.js'; +import { notes } from '$lib/server/db/schema.js'; import { eq, and, desc } from 'drizzle-orm'; import { v4 as uuidv4 } from 'uuid'; import { extractTags } from '$lib/utils/tags.js'; +import { fetchTagsForNotes, syncNoteTags } from '$lib/server/tags.js'; export const GET: RequestHandler = async ({ url }) => { const filter = url.searchParams.get('filter') || 'all'; @@ -21,23 +22,18 @@ export const GET: RequestHandler = async ({ url }) => { conditions = and(eq(notes.archived, false), eq(notes.trashed, false)); } - const result = await db + const result = db .select() .from(notes) .where(conditions) - .orderBy(desc(notes.pinned), desc(notes.updatedAt)); + .orderBy(desc(notes.pinned), desc(notes.updatedAt)) + .all(); - // Fetch tags for each note - const notesWithTags = await Promise.all( - result.map(async (note) => { - const tagRows = await db - .select({ name: tags.name }) - .from(noteTags) - .innerJoin(tags, eq(noteTags.tagId, tags.id)) - .where(eq(noteTags.noteId, note.id)); - return { ...note, tags: tagRows.map((t) => t.name) }; - }) - ); + const tagMap = fetchTagsForNotes(result.map((n) => n.id)); + const notesWithTags = result.map((note) => ({ + ...note, + tags: tagMap.get(note.id) ?? [] + })); return json(notesWithTags); }; @@ -63,27 +59,13 @@ export const POST: RequestHandler = async ({ request }) => { version: 1 }; - await db.insert(notes).values(newNote); - - // Extract and save tags const content = `${newNote.title} ${newNote.content}`; const extractedTags = extractTags(content); - await syncNoteTags(id, extractedTags); + + db.transaction((tx) => { + tx.insert(notes).values(newNote).run(); + }); + syncNoteTags(id, extractedTags); return json({ ...newNote, tags: extractedTags }, { status: 201 }); }; - -async function syncNoteTags(noteId: string, tagNames: string[]) { - // Remove existing tag associations - await db.delete(noteTags).where(eq(noteTags.noteId, noteId)); - - for (const name of tagNames) { - // Upsert tag - let tagRow = await db.select().from(tags).where(eq(tags.name, name)).get(); - if (!tagRow) { - const result = await db.insert(tags).values({ name }).returning(); - tagRow = result[0]; - } - await db.insert(noteTags).values({ noteId, tagId: tagRow.id }); - } -} diff --git a/src/routes/api/notes/[id]/+server.ts b/src/routes/api/notes/[id]/+server.ts index 945be80..918aacc 100644 --- a/src/routes/api/notes/[id]/+server.ts +++ b/src/routes/api/notes/[id]/+server.ts @@ -1,29 +1,26 @@ import { json, error } from '@sveltejs/kit'; import type { RequestHandler } from './$types.js'; import { db } from '$lib/server/db/index.js'; -import { notes, noteTags, tags } from '$lib/server/db/schema.js'; +import { notes } from '$lib/server/db/schema.js'; import { eq } from 'drizzle-orm'; import { extractTags } from '$lib/utils/tags.js'; +import { fetchTagsForNotes, syncNoteTags } from '$lib/server/tags.js'; export const GET: RequestHandler = async ({ params }) => { - const note = await db.select().from(notes).where(eq(notes.id, params.id)).get(); + const note = db.select().from(notes).where(eq(notes.id, params.id)).get(); if (!note) { throw error(404, 'Note not found'); } - const tagRows = await db - .select({ name: tags.name }) - .from(noteTags) - .innerJoin(tags, eq(noteTags.tagId, tags.id)) - .where(eq(noteTags.noteId, note.id)); + const tagMap = fetchTagsForNotes([note.id]); - return json({ ...note, tags: tagRows.map((t) => t.name) }); + return json({ ...note, tags: tagMap.get(note.id) ?? [] }); }; export const PATCH: RequestHandler = async ({ params, request }) => { const body = await request.json(); - const existing = await db.select().from(notes).where(eq(notes.id, params.id)).get(); + const existing = db.select().from(notes).where(eq(notes.id, params.id)).get(); if (!existing) { throw error(404, 'Note not found'); @@ -47,46 +44,28 @@ export const PATCH: RequestHandler = async ({ params, request }) => { if (body.checklistMode !== undefined) updates.checklistMode = body.checklistMode; if (body.sortOrder !== undefined) updates.sortOrder = body.sortOrder; - await db.update(notes).set(updates).where(eq(notes.id, params.id)); + db.update(notes).set(updates).where(eq(notes.id, params.id)).run(); - const updated = await db.select().from(notes).where(eq(notes.id, params.id)).get(); + const updated = db.select().from(notes).where(eq(notes.id, params.id)).get(); - // Re-sync tags if content or title changed if (body.title !== undefined || body.content !== undefined) { const content = `${updated!.title} ${updated!.content}`; const extractedTags = extractTags(content); - await syncNoteTags(params.id, extractedTags); + syncNoteTags(params.id, extractedTags); } - const tagRows = await db - .select({ name: tags.name }) - .from(noteTags) - .innerJoin(tags, eq(noteTags.tagId, tags.id)) - .where(eq(noteTags.noteId, params.id)); + const tagMap = fetchTagsForNotes([params.id]); - return json({ ...updated, tags: tagRows.map((t) => t.name) }); + return json({ ...updated, tags: tagMap.get(params.id) ?? [] }); }; export const DELETE: RequestHandler = async ({ params }) => { - const existing = await db.select().from(notes).where(eq(notes.id, params.id)).get(); + const existing = db.select().from(notes).where(eq(notes.id, params.id)).get(); if (!existing) { throw error(404, 'Note not found'); } - await db.delete(notes).where(eq(notes.id, params.id)); + db.delete(notes).where(eq(notes.id, params.id)).run(); return json({ success: true }); }; - -async function syncNoteTags(noteId: string, tagNames: string[]) { - await db.delete(noteTags).where(eq(noteTags.noteId, noteId)); - - for (const name of tagNames) { - let tagRow = await db.select().from(tags).where(eq(tags.name, name)).get(); - if (!tagRow) { - const result = await db.insert(tags).values({ name }).returning(); - tagRow = result[0]; - } - await db.insert(noteTags).values({ noteId, tagId: tagRow.id }); - } -} diff --git a/src/routes/api/search/+server.ts b/src/routes/api/search/+server.ts index 19f5518..7d7083e 100644 --- a/src/routes/api/search/+server.ts +++ b/src/routes/api/search/+server.ts @@ -2,7 +2,8 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types.js'; import { db } from '$lib/server/db/index.js'; import { notes, noteTags, tags } from '$lib/server/db/schema.js'; -import { and, eq, like, or } from 'drizzle-orm'; +import { and, eq, like, or, inArray } from 'drizzle-orm'; +import { fetchTagsForNotes } from '$lib/server/tags.js'; export const GET: RequestHandler = async ({ url }) => { const query = url.searchParams.get('q')?.trim(); @@ -13,7 +14,7 @@ export const GET: RequestHandler = async ({ url }) => { const pattern = `%${query}%`; - const results = await db + const results = db .select() .from(notes) .where( @@ -21,45 +22,37 @@ export const GET: RequestHandler = async ({ url }) => { eq(notes.trashed, false), or(like(notes.title, pattern), like(notes.content, pattern)) ) - ); + ) + .all(); // Also search by tag name - const tagResults = await db + const tagResults = db .select({ noteId: noteTags.noteId }) .from(noteTags) .innerJoin(tags, eq(noteTags.tagId, tags.id)) - .where(like(tags.name, pattern)); + .where(like(tags.name, pattern)) + .all(); - const tagNoteIds = new Set(tagResults.map((r) => r.noteId)); - const allNoteIds = new Set(results.map((n) => n.id)); + const resultIds = new Set(results.map((n) => n.id)); + const extraNoteIds = [...new Set(tagResults.map((r) => r.noteId))].filter( + (id) => !resultIds.has(id) + ); - // Fetch tag-matched notes not already in results - const extraNoteIds = [...tagNoteIds].filter((id) => !allNoteIds.has(id)); let extraNotes: typeof results = []; if (extraNoteIds.length > 0) { - for (const id of extraNoteIds) { - const note = await db - .select() - .from(notes) - .where(and(eq(notes.id, id), eq(notes.trashed, false))) - .get(); - if (note) extraNotes.push(note); - } + extraNotes = db + .select() + .from(notes) + .where(and(inArray(notes.id, extraNoteIds), eq(notes.trashed, false))) + .all(); } const combined = [...results, ...extraNotes]; - - // Attach tags to each note - const withTags = await Promise.all( - combined.map(async (note) => { - const tagRows = await db - .select({ name: tags.name }) - .from(noteTags) - .innerJoin(tags, eq(noteTags.tagId, tags.id)) - .where(eq(noteTags.noteId, note.id)); - return { ...note, tags: tagRows.map((t) => t.name) }; - }) - ); + const tagMap = fetchTagsForNotes(combined.map((n) => n.id)); + const withTags = combined.map((note) => ({ + ...note, + tags: tagMap.get(note.id) ?? [] + })); return json(withTags); };