Browse Source

merge publication stores

auto-clear cache on new versions
imwald
Silberengel 1 week ago
parent
commit
90b16e845c
  1. 4
      package-lock.json
  2. 2
      package.json
  3. 44
      src/components/CacheBrowser/CacheBrowserDialog.tsx
  4. 75
      src/components/InBrowserCacheSetting/index.tsx
  5. 25
      src/components/VersionUpdateBanner/index.tsx
  6. 4
      src/i18n/locales/en.ts
  7. 98
      src/lib/app-cache-maintenance.ts
  8. 33
      src/lib/library-index-idb-cache.ts
  9. 2
      src/lib/library-publication-index.ts
  10. 732
      src/services/indexed-db.service.ts

4
package-lock.json generated

@ -1,12 +1,12 @@
{ {
"name": "imwald", "name": "imwald",
"version": "23.21.3", "version": "23.21.5",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "imwald", "name": "imwald",
"version": "23.21.3", "version": "23.21.5",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@asciidoctor/core": "^3.0.4", "@asciidoctor/core": "^3.0.4",

2
package.json

@ -1,6 +1,6 @@
{ {
"name": "imwald", "name": "imwald",
"version": "23.21.4", "version": "23.21.5",
"description": "Imwald — a user-friendly Nostr client focused on relay feed browsing, publications, and relay discovery", "description": "Imwald — a user-friendly Nostr client focused on relay feed browsing, publications, and relay discovery",
"private": true, "private": true,
"type": "module", "type": "module",

44
src/components/CacheBrowser/CacheBrowserDialog.tsx

@ -6,7 +6,6 @@ import { Trash2, RefreshCw, Database, WrapText, Search, X, TriangleAlert, Copy,
import { Input } from '@/components/ui/input' import { Input } from '@/components/ui/input'
import { Skeleton } from '@/components/ui/skeleton' import { Skeleton } from '@/components/ui/skeleton'
import indexedDb, { isLikelyCachedNostrEvent, StoreNames, type TCachedEventSearchHit } from '@/services/indexed-db.service' import indexedDb, { isLikelyCachedNostrEvent, StoreNames, type TCachedEventSearchHit } from '@/services/indexed-db.service'
import { clearAllLibraryIndexCaches } from '@/lib/library-publication-index'
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from '@/components/ui/dialog' import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from '@/components/ui/dialog'
import { Drawer, DrawerContent, DrawerHeader, DrawerTitle, DrawerDescription } from '@/components/ui/drawer' import { Drawer, DrawerContent, DrawerHeader, DrawerTitle, DrawerDescription } from '@/components/ui/drawer'
import { HoverCard, HoverCardContent, HoverCardTrigger } from '@/components/ui/hover-card' import { HoverCard, HoverCardContent, HoverCardTrigger } from '@/components/ui/hover-card'
@ -42,6 +41,7 @@ export default function CacheBrowserDialog({
const [globalSearchLoading, setGlobalSearchLoading] = useState(false) const [globalSearchLoading, setGlobalSearchLoading] = useState(false)
const [globalSearchTruncated, setGlobalSearchTruncated] = useState(false) const [globalSearchTruncated, setGlobalSearchTruncated] = useState(false)
const [publicationListFull, setPublicationListFull] = useState(false) const [publicationListFull, setPublicationListFull] = useState(false)
const [publicationStoreTotalRows, setPublicationStoreTotalRows] = useState<number | null>(null)
const globalSearchRequestId = useRef(0) const globalSearchRequestId = useRef(0)
const loadCacheInfo = async () => { const loadCacheInfo = async () => {
@ -63,6 +63,7 @@ export default function CacheBrowserDialog({
setGlobalSearchHits([]) setGlobalSearchHits([])
setGlobalSearchTruncated(false) setGlobalSearchTruncated(false)
setPublicationListFull(false) setPublicationListFull(false)
setPublicationStoreTotalRows(null)
void loadCacheInfo() void loadCacheInfo()
}, [open]) }, [open])
@ -101,13 +102,20 @@ export default function CacheBrowserDialog({
setSelectedStore(storeName) setSelectedStore(storeName)
setSearchQuery('') setSearchQuery('')
setPublicationListFull(false) setPublicationListFull(false)
setPublicationStoreTotalRows(null)
setLoadingItems(true) setLoadingItems(true)
try { try {
const items = if (storeName === StoreNames.PUBLICATION_EVENTS) {
storeName === 'publicationEvents' const [items, allRows] = await Promise.all([
? await indexedDb.getPublicationStoreItems(storeName) indexedDb.getPublicationStoreItems(storeName),
: await indexedDb.getStoreItems(storeName) indexedDb.getStoreItems(storeName)
setStoreItems(items) ])
setPublicationStoreTotalRows(allRows.length)
setStoreItems(items)
} else {
const items = await indexedDb.getStoreItems(storeName)
setStoreItems(items)
}
} catch (error) { } catch (error) {
logger.error('Failed to load store items', { error }) logger.error('Failed to load store items', { error })
toast.error(t('Failed to load store items')) toast.error(t('Failed to load store items'))
@ -178,11 +186,7 @@ export default function CacheBrowserDialog({
if (!selectedStore) return if (!selectedStore) return
if (!confirm(t('Are you sure you want to delete all items from this store?'))) return if (!confirm(t('Are you sure you want to delete all items from this store?'))) return
try { try {
if (selectedStore === StoreNames.LIBRARY_PUBLICATION_INDEX) { await indexedDb.clearStore(selectedStore)
await clearAllLibraryIndexCaches()
} else {
await indexedDb.clearStore(selectedStore)
}
setStoreItems([]) setStoreItems([])
void loadCacheInfo() void loadCacheInfo()
toast.success(t('All items deleted successfully')) toast.success(t('All items deleted successfully'))
@ -199,12 +203,14 @@ export default function CacheBrowserDialog({
setLoadingItems(true) setLoadingItems(true)
try { try {
const result = await indexedDb.cleanupDuplicateReplaceableEvents(selectedStore) const result = await indexedDb.cleanupDuplicateReplaceableEvents(selectedStore)
const items = await indexedDb.getStoreItems(selectedStore)
setStoreItems(items)
setSearchQuery('') setSearchQuery('')
void loadCacheInfo() void loadCacheInfo()
const itemsAfterCleanup = await indexedDb.getStoreItems(selectedStore) const items =
const actualCount = itemsAfterCleanup.length selectedStore === StoreNames.PUBLICATION_EVENTS
? await indexedDb.getPublicationStoreItems(selectedStore)
: await indexedDb.getStoreItems(selectedStore)
setStoreItems(items)
const actualCount = items.length
if (actualCount !== result.kept) { if (actualCount !== result.kept) {
toast.success( toast.success(
t('Cleaned up {{deleted}} duplicate entries, kept {{kept}} (total items after cleanup: {{total}})', { t('Cleaned up {{deleted}} duplicate entries, kept {{kept}} (total items after cleanup: {{total}})', {
@ -218,7 +224,8 @@ export default function CacheBrowserDialog({
} }
} catch (error) { } catch (error) {
logger.error('Failed to cleanup duplicates', { error }) logger.error('Failed to cleanup duplicates', { error })
if (error instanceof Error && error.message === 'Not a replaceable event store') { const message = error instanceof Error ? error.message : String(error)
if (message === 'Not a replaceable event store') {
toast.error(t('This store does not contain replaceable events')) toast.error(t('This store does not contain replaceable events'))
} else { } else {
toast.error(t('Failed to cleanup duplicates')) toast.error(t('Failed to cleanup duplicates'))
@ -465,6 +472,11 @@ export default function CacheBrowserDialog({
<div className="mb-2 flex min-w-0 flex-col gap-3 sm:flex-row sm:items-center sm:justify-between"> <div className="mb-2 flex min-w-0 flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
<div className="min-w-0 text-xs text-muted-foreground"> <div className="min-w-0 text-xs text-muted-foreground">
{filteredStoreItems.length} {t('of')} {storeItems.length} {t('items')} {filteredStoreItems.length} {t('of')} {storeItems.length} {t('items')}
{selectedStore === StoreNames.PUBLICATION_EVENTS &&
publicationStoreTotalRows != null &&
publicationStoreTotalRows > storeItems.length
? ` (${publicationStoreTotalRows} rows incl. nested sections)`
: ''}
{searchQuery.trim() && ` ${t('matching')} "${searchQuery}"`} {searchQuery.trim() && ` ${t('matching')} "${searchQuery}"`}
</div> </div>
<div className="flex min-w-0 flex-shrink-0 flex-wrap gap-2 sm:justify-end"> <div className="flex min-w-0 flex-shrink-0 flex-wrap gap-2 sm:justify-end">

75
src/components/InBrowserCacheSetting/index.tsx

@ -1,4 +1,8 @@
import { Button } from '@/components/ui/button' import { Button } from '@/components/ui/button'
import {
clearAppServiceWorkerAndCaches,
refreshAppBrowserCache
} from '@/lib/app-cache-maintenance'
import { clearConsoleLogBuffer } from '@/lib/console-log-buffer' import { clearConsoleLogBuffer } from '@/lib/console-log-buffer'
import { useConsoleLogBuffer } from '@/hooks/useConsoleLogBuffer' import { useConsoleLogBuffer } from '@/hooks/useConsoleLogBuffer'
import logger from '@/lib/logger' import logger from '@/lib/logger'
@ -15,7 +19,6 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } f
import { Drawer, DrawerContent, DrawerHeader, DrawerTitle, DrawerDescription } from '@/components/ui/drawer' import { Drawer, DrawerContent, DrawerHeader, DrawerTitle, DrawerDescription } from '@/components/ui/drawer'
import { useScreenSize } from '@/providers/ScreenSizeProvider' import { useScreenSize } from '@/providers/ScreenSizeProvider'
import { toast } from 'sonner' import { toast } from 'sonner'
import { syncUserDeletionTombstones } from '@/lib/sync-user-deletions'
import { useCacheBrowser } from '../../contexts/cache-browser-context' import { useCacheBrowser } from '../../contexts/cache-browser-context'
export default function InBrowserCacheSetting() { export default function InBrowserCacheSetting() {
@ -93,11 +96,11 @@ export default function InBrowserCacheSetting() {
const handleRefreshCache = async () => { const handleRefreshCache = async () => {
try { try {
setCacheRefreshBusy(true) setCacheRefreshBusy(true)
await indexedDb.forceDatabaseUpgrade() await refreshAppBrowserCache({
if (pubkey) { pubkey,
await requestAccountNetworkHydrate() relayList,
await syncUserDeletionTombstones(pubkey, relayList) requestAccountNetworkHydrate
} })
toast.success(t('Cache refreshed successfully')) toast.success(t('Cache refreshed successfully'))
} catch (error) { } catch (error) {
logger.error('Failed to refresh cache', { error }) logger.error('Failed to refresh cache', { error })
@ -113,65 +116,7 @@ export default function InBrowserCacheSetting() {
} }
try { try {
const currentOrigin = window.location.origin const { unregisteredCount, cacheClearedCount } = await clearAppServiceWorkerAndCaches()
let unregisteredCount = 0
let cacheClearedCount = 0
if (window.isSecureContext && 'serviceWorker' in navigator) {
let registrations: readonly ServiceWorkerRegistration[] = []
try {
registrations = await navigator.serviceWorker.getRegistrations()
} catch (error) {
logger.warn('Failed to get service worker registrations', { error })
}
if (registrations.length > 0) {
const unregisterPromises = registrations.map(async (registration) => {
try {
const scope = registration.scope
if (scope.startsWith(currentOrigin)) {
const result = await registration.unregister()
if (result) unregisteredCount++
return result
}
return false
} catch (error) {
logger.warn('Failed to unregister a service worker', { error })
return false
}
})
await Promise.all(unregisterPromises)
}
}
if ('caches' in window) {
try {
const cacheNames = await caches.keys()
const appCacheNames = [
'nostr-images',
'satellite-images',
'external-images'
]
const appCaches = cacheNames.filter(name => {
if (appCacheNames.includes(name)) return true
if (name.startsWith('workbox-') || name.startsWith('precache-')) return true
if (name.includes(currentOrigin.replace(/https?:\/\//, '').split('/')[0])) return true
return false
})
await Promise.all(appCaches.map(name => {
cacheClearedCount++
return caches.delete(name).catch(error => {
logger.warn(`Failed to delete cache: ${name}`, { error })
cacheClearedCount--
})
}))
} catch (error) {
logger.warn('Failed to clear some caches', { error })
}
}
if (unregisteredCount > 0 || cacheClearedCount > 0) { if (unregisteredCount > 0 || cacheClearedCount > 0) {
const message = unregisteredCount > 0 && cacheClearedCount > 0 const message = unregisteredCount > 0 && cacheClearedCount > 0

25
src/components/VersionUpdateBanner/index.tsx

@ -1,11 +1,13 @@
import { Button } from '@/components/ui/button' import { Button } from '@/components/ui/button'
import { Skeleton } from '@/components/ui/skeleton' import { Skeleton } from '@/components/ui/skeleton'
import { refreshAppBrowserCacheAndClearServiceWorker } from '@/lib/app-cache-maintenance'
import logger from '@/lib/logger'
import { import {
getPwaApplyUpdate,
initPwaUpdate, initPwaUpdate,
probePwaWaitingWorker, probePwaWaitingWorker,
subscribePwaNeedRefresh subscribePwaNeedRefresh
} from '@/lib/pwa-update' } from '@/lib/pwa-update'
import { useNostrOptional } from '@/providers/nostr-context'
import { RefreshCw, X } from 'lucide-react' import { RefreshCw, X } from 'lucide-react'
import { useEffect, useState } from 'react' import { useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next' import { useTranslation } from 'react-i18next'
@ -21,6 +23,7 @@ function readVersionUpdateDismissed(): boolean {
export default function VersionUpdateBanner() { export default function VersionUpdateBanner() {
const { t } = useTranslation() const { t } = useTranslation()
const nostr = useNostrOptional()
const [updateAvailable, setUpdateAvailable] = useState(false) const [updateAvailable, setUpdateAvailable] = useState(false)
const [isDismissed, setIsDismissed] = useState(readVersionUpdateDismissed) const [isDismissed, setIsDismissed] = useState(readVersionUpdateDismissed)
const [isUpdating, setIsUpdating] = useState(false) const [isUpdating, setIsUpdating] = useState(false)
@ -65,16 +68,18 @@ export default function VersionUpdateBanner() {
setIsDismissed(true) setIsDismissed(true)
setIsUpdating(true) setIsUpdating(true)
const reload = () => { void (async () => {
try {
await refreshAppBrowserCacheAndClearServiceWorker({
pubkey: nostr?.pubkey,
relayList: nostr?.relayList,
requestAccountNetworkHydrate: nostr?.requestAccountNetworkHydrate
})
} catch (error) {
logger.warn('[VersionUpdateBanner] Pre-update cache refresh failed', { error })
}
window.location.reload() window.location.reload()
} })()
const apply = getPwaApplyUpdate()
if (apply) {
void apply().catch(reload)
return
}
reload()
} }
const handleDismiss = () => { const handleDismiss = () => {

4
src/i18n/locales/en.ts

@ -1708,7 +1708,7 @@ export default {
'Read this book': 'Read this book', 'Read this book': 'Read this book',
'libraryIndexCache.sectionTitle': 'Library publication index', 'libraryIndexCache.sectionTitle': 'Library publication index',
'libraryIndexCache.sectionBlurb': 'libraryIndexCache.sectionBlurb':
'Cached kind-30040 index events used to populate the Library tab. Clearing this only removes the discovery list cache—not publications you have opened for reading.', 'Kind-30040 catalog rows in your publication cache (same store as opened books). Clearing removes relay-discovered titles only—publications you have opened stay cached with their sections.',
'libraryIndexCache.defaultsMobile': 'libraryIndexCache.defaultsMobile':
'Default on mobile web: up to {{entries}} indexes, ~{{mb}} MB.', 'Default on mobile web: up to {{entries}} indexes, ~{{mb}} MB.',
'libraryIndexCache.defaultsElectron': 'libraryIndexCache.defaultsElectron':
@ -1720,7 +1720,7 @@ export default {
'libraryIndexCache.clear': 'Clear library index cache', 'libraryIndexCache.clear': 'Clear library index cache',
'libraryIndexCache.clearing': 'Clearing…', 'libraryIndexCache.clearing': 'Clearing…',
'libraryIndexCache.clearConfirm': 'libraryIndexCache.clearConfirm':
'Clear the Library index cache? The Library tab will reload indexes from relays on next visit. Opened publications stay in your publication reading cache.', 'Clear relay-discovered library catalog entries? Opened publications and their sections stay in the publication cache.',
'libraryIndexCache.clearedToast': 'Library index cache cleared.', 'libraryIndexCache.clearedToast': 'Library index cache cleared.',
'libraryIndexCache.clearFailed': 'Failed to clear library index cache.', 'libraryIndexCache.clearFailed': 'Failed to clear library index cache.',
'Search page clear': 'Clear', 'Search page clear': 'Clear',

98
src/lib/app-cache-maintenance.ts

@ -0,0 +1,98 @@
import logger from '@/lib/logger'
import { syncUserDeletionTombstones } from '@/lib/sync-user-deletions'
import indexedDb from '@/services/indexed-db.service'
import type { TRelayList } from '@/types'
const APP_CACHE_NAMES = ['nostr-images', 'satellite-images', 'external-images'] as const
function isAppRelatedCacheName(name: string, origin: string): boolean {
if ((APP_CACHE_NAMES as readonly string[]).includes(name)) return true
if (name.startsWith('workbox-') || name.startsWith('precache-')) return true
const host = origin.replace(/https?:\/\//, '').split('/')[0]
return name.includes(host)
}
export type ClearAppServiceWorkerResult = {
unregisteredCount: number
cacheClearedCount: number
}
export async function clearAppPrecacheCaches(): Promise<number> {
if (typeof window === 'undefined' || !('caches' in window)) return 0
let cleared = 0
try {
const cacheNames = await caches.keys()
const origin = window.location.origin
const appCaches = cacheNames.filter((name) => isAppRelatedCacheName(name, origin))
await Promise.all(
appCaches.map((name) =>
caches
.delete(name)
.then((ok) => {
if (ok) cleared++
})
.catch((error) => {
logger.warn('[app-cache] Failed to delete cache', { name, error })
})
)
)
} catch (error) {
logger.warn('[app-cache] Failed to clear precache caches', { error })
}
return cleared
}
export async function unregisterAppServiceWorkers(): Promise<number> {
if (typeof window === 'undefined' || !window.isSecureContext || !('serviceWorker' in navigator)) {
return 0
}
let count = 0
try {
const registrations = await navigator.serviceWorker.getRegistrations()
const origin = window.location.origin
await Promise.all(
registrations.map(async (registration) => {
if (!registration.scope.startsWith(origin)) return
try {
if (await registration.unregister()) count++
} catch (error) {
logger.warn('[app-cache] Failed to unregister service worker', { error })
}
})
)
} catch (error) {
logger.warn('[app-cache] Failed to get service worker registrations', { error })
}
return count
}
export async function clearAppServiceWorkerAndCaches(): Promise<ClearAppServiceWorkerResult> {
const cacheClearedCount = await clearAppPrecacheCaches()
const unregisteredCount = await unregisterAppServiceWorkers()
return { unregisteredCount, cacheClearedCount }
}
export type RefreshAppBrowserCacheOptions = {
pubkey?: string | null
relayList?: TRelayList | null
requestAccountNetworkHydrate?: () => Promise<void>
}
export async function refreshAppBrowserCache(options?: RefreshAppBrowserCacheOptions): Promise<void> {
await indexedDb.forceDatabaseUpgrade()
const pubkey = options?.pubkey?.trim()
if (pubkey && options?.requestAccountNetworkHydrate) {
await options.requestAccountNetworkHydrate()
await syncUserDeletionTombstones(pubkey, options.relayList ?? null)
}
}
/** IndexedDB refresh + service worker unregister + Cache API clear (settings / update banner). */
export async function refreshAppBrowserCacheAndClearServiceWorker(
options?: RefreshAppBrowserCacheOptions
): Promise<ClearAppServiceWorkerResult> {
await refreshAppBrowserCache(options)
return clearAppServiceWorkerAndCaches()
}

33
src/lib/library-index-idb-cache.ts

@ -12,30 +12,22 @@ import indexedDb from '@/services/indexed-db.service'
import type { Event } from 'nostr-tools' import type { Event } from 'nostr-tools'
type PersistLibraryIndexCacheOptions = { type PersistLibraryIndexCacheOptions = {
/** When false, merge rows only — skip reconcile so partial batches cannot wipe unrelated cache rows. */ /** Accepted for API compat; reconcile is a no-op after v43 store consolidation. */
reconcile?: boolean reconcile?: boolean
} }
let persistQueue: Promise<void> = Promise.resolve() let persistQueue: Promise<void> = Promise.resolve()
/** Load kind-30040 catalog masters from {@link StoreNames.PUBLICATION_EVENTS}. */
export async function loadLibraryIndexCacheEvents(): Promise<Event[]> { export async function loadLibraryIndexCacheEvents(): Promise<Event[]> {
try { try {
const cached = await indexedDb.getLibraryPublicationIndexCacheEvents() const cached = await indexedDb.getPublicationCatalogIndexEvents()
// Structural re-check + address dedupe only — avoid ~5k verifyEvent on read (main-thread hang).
const structural = filterStructuralIndexEvents(cached) const structural = filterStructuralIndexEvents(cached)
const map = buildStructuralPublicationIndexMap(structural) const map = buildStructuralPublicationIndexMap(structural)
const normalized = publicationIndexMapValues(map) return publicationIndexMapValues(map)
if (structural.length < cached.length) {
void indexedDb.pruneUnverifiedLibraryPublicationIndexCacheEvents().catch(() => {})
}
const hasLegacyKeys = await indexedDb.libraryPublicationIndexCacheHasLegacyKeys()
if (normalized.length !== cached.length || hasLegacyKeys) {
void persistLibraryIndexCacheEvents(normalized, { reconcile: true }).catch(() => {})
}
return normalized
} catch (e) { } catch (e) {
if (import.meta.env.DEV) { if (import.meta.env.DEV) {
logger.warn('[Library] index IDB read failed', { logger.warn('[Library] publication catalog IDB read failed', {
message: e instanceof Error ? e.message : String(e) message: e instanceof Error ? e.message : String(e)
}) })
} }
@ -43,25 +35,22 @@ export async function loadLibraryIndexCacheEvents(): Promise<Event[]> {
} }
} }
/** Persist kind-30040 catalog masters into {@link StoreNames.PUBLICATION_EVENTS}. */
export async function persistLibraryIndexCacheEvents( export async function persistLibraryIndexCacheEvents(
events: Event[], events: Event[],
options?: PersistLibraryIndexCacheOptions _options?: PersistLibraryIndexCacheOptions
): Promise<void> { ): Promise<void> {
const map = buildStructuralPublicationIndexMap(filterStructuralIndexEvents(events)) const map = buildStructuralPublicationIndexMap(filterStructuralIndexEvents(events))
const normalized = publicationIndexMapValues(map) const normalized = publicationIndexMapValues(map)
if (normalized.length === 0) return if (normalized.length === 0) return
const reconcile = options?.reconcile !== false
const run = async () => { const run = async () => {
try { try {
const budget = getLibraryIndexCacheBudget() const budget = getLibraryIndexCacheBudget()
await indexedDb.mergeLibraryPublicationIndexCacheEvents(normalized, budget) await indexedDb.mergePublicationCatalogIndexEvents(normalized, budget)
if (reconcile) {
await indexedDb.reconcileLibraryPublicationIndexCache(map)
}
} catch (e) { } catch (e) {
if (import.meta.env.DEV) { if (import.meta.env.DEV) {
logger.warn('[Library] index IDB write failed', { logger.warn('[Library] publication catalog IDB write failed', {
message: e instanceof Error ? e.message : String(e) message: e instanceof Error ? e.message : String(e)
}) })
} }
@ -74,14 +63,14 @@ export async function persistLibraryIndexCacheEvents(
export async function getLibraryIndexCacheFootprint(): Promise<{ count: number; bytes: number }> { export async function getLibraryIndexCacheFootprint(): Promise<{ count: number; bytes: number }> {
try { try {
return await indexedDb.getLibraryPublicationIndexCacheFootprint() return await indexedDb.getPublicationCatalogFootprint()
} catch { } catch {
return { count: 0, bytes: 0 } return { count: 0, bytes: 0 }
} }
} }
export async function clearLibraryIndexIdbCache(): Promise<void> { export async function clearLibraryIndexIdbCache(): Promise<void> {
await indexedDb.clearLibraryPublicationIndexCacheStore() await indexedDb.clearPublicationCatalogDiscoveryOnly()
} }
export { approxLibraryIndexEventBytes, getLibraryIndexCacheBudget } export { approxLibraryIndexEventBytes, getLibraryIndexCacheBudget }

2
src/lib/library-publication-index.ts

@ -2210,7 +2210,7 @@ export function clearLibraryPublicationIndexCache(): void {
clearLibrarySearchSessionCache() clearLibrarySearchSessionCache()
} }
/** Clears Library tab session + IDB index cache only (publication reading cache is unchanged). */ /** Clears Library tab session cache and relay-discovered catalog masters (opened publications stay). */
export async function clearAllLibraryIndexCaches(): Promise<void> { export async function clearAllLibraryIndexCaches(): Promise<void> {
sessionCache = null sessionCache = null
indexLoadJob = null indexLoadJob = null

732
src/services/indexed-db.service.ts

@ -1,7 +1,11 @@
import { CALENDAR_EVENT_KINDS, ExtendedKind } from '@/constants' import { CALENDAR_EVENT_KINDS, ExtendedKind } from '@/constants'
/** Legacy object store names removed in DB migrations (do not re-add to {@link StoreNames}). */ /** Legacy object store names removed in DB migrations (do not re-add to {@link StoreNames}). */
const LEGACY_DELETED_OBJECT_STORES = ['relayInfoEvents', 'spellListSourceEvents'] as const const LEGACY_DELETED_OBJECT_STORES = [
'relayInfoEvents',
'spellListSourceEvents',
'libraryPublicationIndex'
] as const
import { import {
publicationCoordinateLookupKeys, publicationCoordinateLookupKeys,
splitPublicationCoordinate splitPublicationCoordinate
@ -28,9 +32,7 @@ import logger from '@/lib/logger'
import { profileKind0MatchesSearchQuery } from '@/lib/profile-metadata-search' import { profileKind0MatchesSearchQuery } from '@/lib/profile-metadata-search'
import { shouldDropEventOnIngest } from '@/lib/event-ingest-filter' import { shouldDropEventOnIngest } from '@/lib/event-ingest-filter'
import { import {
eventTagAddress,
isStructuralPublicationIndex, isStructuralPublicationIndex,
isVerifiedPublicationIndex,
pickNewerPublicationIndexEvent, pickNewerPublicationIndexEvent,
type PublicationIndexMap type PublicationIndexMap
} from '@/lib/publication-index' } from '@/lib/publication-index'
@ -71,6 +73,12 @@ type TValue<T = any> = {
value: T | null value: T | null
addedAt: number addedAt: number
masterPublicationKey?: string // For nested publication events, link to master publication masterPublicationKey?: string // For nested publication events, link to master publication
/** 1 when `value` is a kind-30040 library catalog master (indexed for Library tab). */
catalogMaster?: 0 | 1
/** LRU touch time for catalog masters in {@link StoreNames.PUBLICATION_EVENTS}. */
lastAccessAt?: number
/** Approximate JSON size for catalog-master LRU pruning. */
catalogBytes?: number
} }
/** One matching row from {@link IndexedDbService.searchAllCachedEventsFullText}. */ /** One matching row from {@link IndexedDbService.searchAllCachedEventsFullText}. */
@ -150,7 +158,6 @@ export const StoreNames = {
/** Piper / read-aloud WAV blobs keyed by SHA-256 of endpoint + text + speed. */ /** Piper / read-aloud WAV blobs keyed by SHA-256 of endpoint + text + speed. */
PIPER_TTS_CACHE: 'piperTtsCache', PIPER_TTS_CACHE: 'piperTtsCache',
/** Library kind-30040 index LRU (rotating cache; separate byte/entry budget from EVENT_ARCHIVE). */ /** Library kind-30040 index LRU (rotating cache; separate byte/entry budget from EVENT_ARCHIVE). */
LIBRARY_PUBLICATION_INDEX: 'libraryPublicationIndex',
/** NIP-52 calendar notes (31922/31923). Key: {@link replaceableEventDedupeKey}. Index: `occurrenceStartMs`. */ /** NIP-52 calendar notes (31922/31923). Key: {@link replaceableEventDedupeKey}. Index: `occurrenceStartMs`. */
CALENDAR_EVENTS: 'calendarEvents', CALENDAR_EVENTS: 'calendarEvents',
/** NIP-52 calendar RSVPs (31925). Key: event id. Index: `parentCoordinate` (`a` tag). */ /** NIP-52 calendar RSVPs (31925). Key: event id. Index: `parentCoordinate` (`a` tag). */
@ -182,7 +189,6 @@ export type TCalendarRsvpCacheRow = {
const CACHE_BROWSER_EVENT_SEARCH_EXCLUDED_STORES: ReadonlySet<string> = new Set([ const CACHE_BROWSER_EVENT_SEARCH_EXCLUDED_STORES: ReadonlySet<string> = new Set([
StoreNames.SETTINGS, StoreNames.SETTINGS,
StoreNames.PIPER_TTS_CACHE, StoreNames.PIPER_TTS_CACHE,
StoreNames.LIBRARY_PUBLICATION_INDEX,
StoreNames.RELAY_INFOS, StoreNames.RELAY_INFOS,
StoreNames.NIP66_DISCOVERY, StoreNames.NIP66_DISCOVERY,
StoreNames.GIF_CACHE, StoreNames.GIF_CACHE,
@ -237,7 +243,7 @@ const FULL_TEXT_NOTE_SEARCH_STORES: ReadonlySet<string> = new Set([
const ARCHIVE_CALENDAR_PURGE_SETTING_KEY = 'archiveCalendarPurgedV37' const ARCHIVE_CALENDAR_PURGE_SETTING_KEY = 'archiveCalendarPurgedV37'
/** Schema version we expect. When adding stores or migrations, bump this. */ /** Schema version we expect. When adding stores or migrations, bump this. */
const DB_VERSION = 41 const DB_VERSION = 43
/** Hint age for profile/payment reads (stale rows still returned; background refresh). */ /** Hint age for profile/payment reads (stale rows still returned; background refresh). */
const PROFILE_AND_PAYMENT_STALE_READ_MS = 5 * 60 * 1000 const PROFILE_AND_PAYMENT_STALE_READ_MS = 5 * 60 * 1000
@ -253,7 +259,7 @@ function idbEventToError(ev: Parameters<NonNullable<IDBRequest['onerror']>>[0]):
return new Error(message) return new Error(message)
} }
type TLibraryPublicationIndexCacheRow = { type TLegacyLibraryPublicationIndexRow = {
key: string key: string
value: Event value: Event
addedAt: number addedAt: number
@ -261,7 +267,7 @@ type TLibraryPublicationIndexCacheRow = {
approxBytes: number approxBytes: number
} }
function approxLibraryPublicationIndexRowBytes(ev: Event): number { function approxPublicationCatalogMasterBytes(ev: Event): number {
try { try {
return new Blob([JSON.stringify(ev)]).size return new Blob([JSON.stringify(ev)]).size
} catch { } catch {
@ -269,53 +275,125 @@ function approxLibraryPublicationIndexRowBytes(ev: Event): number {
} }
} }
/** v41: re-key library index rows from event id to kind:pubkey:d; dedupe by address. */ function publicationStoreRowKeyForEvent(event: Event): string {
function migrateLibraryPublicationIndexCacheToAddressKeys(transaction: IDBTransaction): void { const [, d] = event.tags.find(tagNameEquals('d')) ?? []
const store = transaction.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX) const trimmed = event.pubkey.trim()
const rows: TLibraryPublicationIndexCacheRow[] = [] const canonPk = /^[0-9a-f]{64}$/i.test(trimmed) ? trimmed.toLowerCase() : trimmed
return d === undefined ? canonPk : `${canonPk}:${d}`
}
function buildPublicationStoreRow(
key: string,
event: Event,
prev: TValue<Event> | undefined,
masterPublicationKey?: string
): TValue<Event> {
const now = Date.now()
const isCatalogMaster = event.kind === ExtendedKind.PUBLICATION && !masterPublicationKey
const row: TValue<Event> = {
key,
value: event,
addedAt: prev?.addedAt ?? now,
...(masterPublicationKey ? { masterPublicationKey } : {})
}
if (isCatalogMaster) {
row.catalogMaster = 1
row.lastAccessAt = Math.max(prev?.lastAccessAt ?? 0, now)
row.catalogBytes = approxPublicationCatalogMasterBytes(event)
} else if (storeRowIsPublicationEvent(event)) {
row.catalogMaster = 0
}
return row
}
const readReq = store.openCursor() function storeRowIsPublicationEvent(event: Event): boolean {
return (
event.kind === ExtendedKind.PUBLICATION ||
event.kind === ExtendedKind.PUBLICATION_CONTENT ||
event.kind === ExtendedKind.WIKI_ARTICLE ||
event.kind === ExtendedKind.NOSTR_SPECIFICATION ||
event.kind === kinds.LongFormArticle
)
}
function ensurePublicationEventsCatalogIndexes(store: IDBObjectStore): void {
if (!store.indexNames.contains('catalogMaster')) {
store.createIndex('catalogMaster', 'catalogMaster', { unique: false })
}
}
function backfillPublicationCatalogMetadata(store: IDBObjectStore): void {
const req = store.openCursor()
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) return
const row = cursor.value as TValue<Event>
const event = row?.value
if (!event || !storeRowIsPublicationEvent(event)) {
cursor.continue()
return
}
const isCatalogMaster = event.kind === ExtendedKind.PUBLICATION && !row.masterPublicationKey
const next: TValue<Event> = { ...row }
if (isCatalogMaster) {
next.catalogMaster = 1
next.lastAccessAt = row.lastAccessAt ?? row.addedAt ?? Date.now()
next.catalogBytes = row.catalogBytes ?? approxPublicationCatalogMasterBytes(event)
} else {
next.catalogMaster = 0
}
if (
next.catalogMaster !== row.catalogMaster ||
next.lastAccessAt !== row.lastAccessAt ||
next.catalogBytes !== row.catalogBytes
) {
const updateReq = cursor.update(next)
updateReq.onsuccess = () => cursor.continue()
updateReq.onerror = () => cursor.continue()
} else {
cursor.continue()
}
}
}
/** v43: merge legacy libraryPublicationIndex into publicationEvents; catalogMaster index. */
function migrateLegacyLibraryPublicationIndexStore(transaction: IDBTransaction, db: IDBDatabase): void {
const legacyName = 'libraryPublicationIndex'
if (!db.objectStoreNames.contains(legacyName) || !db.objectStoreNames.contains(StoreNames.PUBLICATION_EVENTS)) {
return
}
const legacyStore = transaction.objectStore(legacyName)
const pubStore = transaction.objectStore(StoreNames.PUBLICATION_EVENTS)
const legacyRows: TLegacyLibraryPublicationIndexRow[] = []
const readReq = legacyStore.openCursor()
readReq.onsuccess = () => { readReq.onsuccess = () => {
const cursor = readReq.result const cursor = readReq.result as IDBCursorWithValue | null
if (cursor) { if (cursor) {
rows.push(cursor.value as TLibraryPublicationIndexCacheRow) legacyRows.push(cursor.value as TLegacyLibraryPublicationIndexRow)
cursor.continue() cursor.continue()
return return
} }
const byAddress = new Map<string, TLibraryPublicationIndexCacheRow>() for (const row of legacyRows) {
for (const row of rows) {
const ev = row?.value const ev = row?.value
if (!ev || ev.kind !== ExtendedKind.PUBLICATION || !isStructuralPublicationIndex(ev)) continue if (!ev || ev.kind !== ExtendedKind.PUBLICATION || !isStructuralPublicationIndex(ev)) continue
const addr = eventTagAddress(ev) const key = publicationStoreRowKeyForEvent(ev)
if (!addr) continue const getReq = pubStore.get(key)
getReq.onsuccess = () => {
const existing = byAddress.get(addr) const prev = getReq.result as TValue<Event> | undefined
if (!existing) { const winner =
byAddress.set(addr, { prev?.value && pickNewerPublicationIndexEvent(prev.value, ev).id === prev.value.id
key: addr, ? prev.value
value: ev, : ev
addedAt: row.addedAt ?? Date.now(), const merged: TValue<Event> = buildPublicationStoreRow(key, winner, prev)
lastAccessAt: row.lastAccessAt ?? row.addedAt ?? Date.now(), merged.addedAt = Math.min(prev?.addedAt ?? row.addedAt ?? Date.now(), row.addedAt ?? Date.now())
approxBytes: row.approxBytes ?? approxLibraryPublicationIndexRowBytes(ev) merged.lastAccessAt = Math.max(
}) prev?.lastAccessAt ?? 0,
continue row.lastAccessAt ?? row.addedAt ?? 0
} )
merged.catalogBytes = approxPublicationCatalogMasterBytes(winner)
const winner = pickNewerPublicationIndexEvent(existing.value, ev) pubStore.put(merged)
byAddress.set(addr, {
key: addr,
value: winner,
addedAt: Math.min(existing.addedAt, row.addedAt ?? existing.addedAt),
lastAccessAt: Math.max(existing.lastAccessAt, row.lastAccessAt ?? row.addedAt ?? 0),
approxBytes: approxLibraryPublicationIndexRowBytes(winner)
})
}
const clearReq = store.clear()
clearReq.onsuccess = () => {
for (const row of byAddress.values()) {
store.put(row)
} }
} }
} }
@ -347,9 +425,11 @@ function ensureMissingObjectStores(db: IDBDatabase): void {
const pa = db.createObjectStore(storeName, { keyPath: 'key' }) const pa = db.createObjectStore(storeName, { keyPath: 'key' })
pa.createIndex('authorPubkey', 'authorPubkey', { unique: false }) pa.createIndex('authorPubkey', 'authorPubkey', { unique: false })
pa.createIndex('targetEventId', 'targetEventId', { unique: false }) pa.createIndex('targetEventId', 'targetEventId', { unique: false })
} else if (storeName === StoreNames.LIBRARY_PUBLICATION_INDEX) { } else if (storeName === StoreNames.PUBLICATION_EVENTS) {
const lib = db.createObjectStore(storeName, { keyPath: 'key' }) const store = db.createObjectStore(storeName, { keyPath: 'key' })
lib.createIndex('lastAccessAt', 'lastAccessAt', { unique: false }) ensurePublicationEventsCatalogIndexes(store)
} else if (storeName === 'libraryPublicationIndex') {
/* dropped in v43 — do not recreate */
} else { } else {
db.createObjectStore(storeName, { keyPath: 'key' }) db.createObjectStore(storeName, { keyPath: 'key' })
} }
@ -514,7 +594,8 @@ class IndexedDbService {
db.createObjectStore(StoreNames.RELAY_INFOS, { keyPath: 'key' }) db.createObjectStore(StoreNames.RELAY_INFOS, { keyPath: 'key' })
} }
if (!db.objectStoreNames.contains(StoreNames.PUBLICATION_EVENTS)) { if (!db.objectStoreNames.contains(StoreNames.PUBLICATION_EVENTS)) {
db.createObjectStore(StoreNames.PUBLICATION_EVENTS, { keyPath: 'key' }) const pub = db.createObjectStore(StoreNames.PUBLICATION_EVENTS, { keyPath: 'key' })
ensurePublicationEventsCatalogIndexes(pub)
} }
if (!db.objectStoreNames.contains(StoreNames.PUBLIC_LIVELY_RELAYS)) { if (!db.objectStoreNames.contains(StoreNames.PUBLIC_LIVELY_RELAYS)) {
db.createObjectStore(StoreNames.PUBLIC_LIVELY_RELAYS, { keyPath: 'key' }) db.createObjectStore(StoreNames.PUBLIC_LIVELY_RELAYS, { keyPath: 'key' })
@ -591,15 +672,29 @@ class IndexedDbService {
} }
} }
if (event.oldVersion < 40) { if (event.oldVersion < 40) {
if (!db.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) { if (!db.objectStoreNames.contains('libraryPublicationIndex')) {
const lib = db.createObjectStore(StoreNames.LIBRARY_PUBLICATION_INDEX, { keyPath: 'key' }) const lib = db.createObjectStore('libraryPublicationIndex', { keyPath: 'key' })
lib.createIndex('lastAccessAt', 'lastAccessAt', { unique: false }) lib.createIndex('lastAccessAt', 'lastAccessAt', { unique: false })
} }
} }
if (event.oldVersion < 41) { if (event.oldVersion < 41) {
const tx = (event.target as IDBOpenDBRequest).transaction const tx = (event.target as IDBOpenDBRequest).transaction
if (tx && db.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) { if (tx && db.objectStoreNames.contains('libraryPublicationIndex')) {
migrateLibraryPublicationIndexCacheToAddressKeys(tx) // v41 migration superseded by v43 consolidation into publicationEvents
}
}
if (event.oldVersion < 43) {
const tx = (event.target as IDBOpenDBRequest).transaction
if (tx && db.objectStoreNames.contains(StoreNames.PUBLICATION_EVENTS)) {
const pubStore = tx.objectStore(StoreNames.PUBLICATION_EVENTS)
ensurePublicationEventsCatalogIndexes(pubStore)
backfillPublicationCatalogMetadata(pubStore)
}
if (tx) {
migrateLegacyLibraryPublicationIndexStore(tx, db)
}
if (db.objectStoreNames.contains('libraryPublicationIndex')) {
db.deleteObjectStore('libraryPublicationIndex')
} }
} }
ensureMissingObjectStores(db) ensureMissingObjectStores(db)
@ -690,7 +785,11 @@ class IndexedDbService {
return resolve(oldValue.value) return resolve(oldValue.value)
} }
const putRequest = store.put(this.formatValue(key, cleanEvent)) const putRequest = store.put(
storeName === StoreNames.PUBLICATION_EVENTS
? buildPublicationStoreRow(key, cleanEvent, oldValue)
: this.formatValue(key, cleanEvent)
)
putRequest.onsuccess = () => { putRequest.onsuccess = () => {
transaction.commit() transaction.commit()
resolve(cleanEvent) resolve(cleanEvent)
@ -1385,16 +1484,14 @@ class IndexedDbService {
if (oldValue?.value && oldValue.value.created_at > cleanEvent.created_at) { if (oldValue?.value && oldValue.value.created_at > cleanEvent.created_at) {
// Update master key link even if event is not newer // Update master key link even if event is not newer
if (oldValue.masterPublicationKey !== masterKey) { if (oldValue.masterPublicationKey !== masterKey) {
const value = this.formatValue(key, oldValue.value) const value = buildPublicationStoreRow(key, oldValue.value, oldValue, masterKey)
value.masterPublicationKey = masterKey
store.put(value) store.put(value)
} }
transaction.commit() transaction.commit()
return resolve(oldValue.value) return resolve(oldValue.value)
} }
// Store with master key link // Store with master key link
const value = this.formatValue(key, cleanEvent) const value = buildPublicationStoreRow(key, cleanEvent, oldValue, masterKey)
value.masterPublicationKey = masterKey
const putRequest = store.put(value) const putRequest = store.put(value)
putRequest.onsuccess = () => { putRequest.onsuccess = () => {
transaction.commit() transaction.commit()
@ -1442,8 +1539,7 @@ class IndexedDbService {
// For non-replaceable events, use event ID as key // For non-replaceable events, use event ID as key
const key = event.id const key = event.id
// For non-replaceable events, always update with master key link // For non-replaceable events, always update with master key link
const value = this.formatValue(key, event) const value = buildPublicationStoreRow(key, event, undefined, masterKey)
value.masterPublicationKey = masterKey
const putRequest = store.put(value) const putRequest = store.put(value)
putRequest.onsuccess = () => { putRequest.onsuccess = () => {
transaction.commit() transaction.commit()
@ -2080,7 +2176,7 @@ class IndexedDbService {
storeInfo[storeName] = req.result storeInfo[storeName] = req.result
pending-- pending--
if (pending === 0) { if (pending === 0) {
resolve(storeInfo) void this.normalizePublicationStoreInfoCount(storeInfo).then(resolve).catch(reject)
} }
} }
req.onerror = (ev) => { req.onerror = (ev) => {
@ -2090,6 +2186,116 @@ class IndexedDbService {
}) })
} }
/** Master kind-30040 catalog rows in {@link StoreNames.PUBLICATION_EVENTS}. */
async countPublicationStoreMasterEvents(): Promise<number> {
await this.initPromise
if (!this.db?.objectStoreNames.contains(StoreNames.PUBLICATION_EVENTS)) return 0
return new Promise((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.PUBLICATION_EVENTS, 'readonly')
const store = tx.objectStore(StoreNames.PUBLICATION_EVENTS)
if (store.indexNames.contains('catalogMaster')) {
const req = store.index('catalogMaster').count(IDBKeyRange.only(1))
req.onsuccess = () => {
tx.commit()
resolve(req.result)
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
return
}
const req = store.openCursor()
let count = 0
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve(count)
return
}
const item = cursor.value as TValue<Event> | undefined
if (item?.catalogMaster === 1 || (item?.value?.kind === ExtendedKind.PUBLICATION && !item.masterPublicationKey)) {
count += 1
}
cursor.continue()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
})
}
/** Kind-30040 library catalog masters in {@link StoreNames.PUBLICATION_EVENTS}. */
async getPublicationCatalogIndexEvents(): Promise<Event[]> {
await this.initPromise
if (!this.db?.objectStoreNames.contains(StoreNames.PUBLICATION_EVENTS)) return []
return new Promise((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.PUBLICATION_EVENTS, 'readonly')
const store = tx.objectStore(StoreNames.PUBLICATION_EVENTS)
const out: Event[] = []
const collectRow = (row: TValue<Event> | undefined) => {
const event = row?.value
if (!event || event.kind !== ExtendedKind.PUBLICATION) return
if (row?.catalogMaster === 1 || !row.masterPublicationKey) out.push(event)
}
if (store.indexNames.contains('catalogMaster')) {
const req = store.index('catalogMaster').openCursor(IDBKeyRange.only(1))
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve(out)
return
}
collectRow(cursor.value as TValue<Event>)
cursor.continue()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
return
}
const req = store.openCursor()
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve(out)
return
}
collectRow(cursor.value as TValue<Event>)
cursor.continue()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
})
}
/** @deprecated Use {@link getPublicationCatalogIndexEvents}. */
async getMasterPublicationIndexEventsFromReadingCache(): Promise<Event[]> {
return this.getPublicationCatalogIndexEvents()
}
private async normalizePublicationStoreInfoCount(storeInfo: Record<string, number>): Promise<Record<string, number>> {
if (!storeInfo[StoreNames.PUBLICATION_EVENTS]) return storeInfo
try {
storeInfo[StoreNames.PUBLICATION_EVENTS] = await this.countPublicationStoreMasterEvents()
} catch (e) {
logger.warn('[indexedDb] countPublicationStoreMasterEvents failed', { e })
}
return storeInfo
}
async getStoreItems(storeName: string): Promise<TValue<any>[]> { async getStoreItems(storeName: string): Promise<TValue<any>[]> {
await this.initPromise await this.initPromise
if (!this.db || !this.db.objectStoreNames.contains(storeName)) { if (!this.db || !this.db.objectStoreNames.contains(storeName)) {
@ -2287,13 +2493,27 @@ class IndexedDbService {
return { deleted: 0, kept: 0 } return { deleted: 0, kept: 0 }
} }
if (storeName === StoreNames.PUBLICATION_EVENTS) {
return this.cleanupDuplicateEventsByDedupeKey(storeName, (event) =>
this.getReplaceableEventKeyFromEvent(event)
)
}
// Get the kind for this store - only clean up replaceable event stores // Get the kind for this store - only clean up replaceable event stores
const kind = this.getKindByStoreName(storeName) const kind = this.getKindByStoreName(storeName)
if (!kind || !this.isReplaceableEventKind(kind)) { if (!kind || !this.isReplaceableEventKind(kind)) {
return Promise.reject('Not a replaceable event store') return Promise.reject(new Error('Not a replaceable event store'))
} }
// First pass: identify duplicates return this.cleanupDuplicateEventsByDedupeKey(storeName, (event) =>
this.getReplaceableEventKeyFromEvent(event)
)
}
private async cleanupDuplicateEventsByDedupeKey(
storeName: string,
dedupeKeyForEvent: (event: Event) => string
): Promise<{ deleted: number; kept: number }> {
const allItems = await this.getStoreItems(storeName) const allItems = await this.getStoreItems(storeName)
const eventMap = new Map<string, { key: string; event: Event; addedAt: number }>() const eventMap = new Map<string, { key: string; event: Event; addedAt: number }>()
const keysToDelete: string[] = [] const keysToDelete: string[] = []
@ -2305,47 +2525,41 @@ class IndexedDbService {
continue continue
} }
// Skip if event doesn't have required fields
if (!item.value.pubkey || !item.value.kind || !item.value.created_at) { if (!item.value.pubkey || !item.value.kind || !item.value.created_at) {
invalidItemsCount++ invalidItemsCount++
continue continue
} }
try { try {
const replaceableKey = this.getReplaceableEventKeyFromEvent(item.value) const dedupeKey = dedupeKeyForEvent(item.value)
const existing = eventMap.get(replaceableKey) const existing = eventMap.get(dedupeKey)
if (!existing || if (
item.value.created_at > existing.event.created_at || !existing ||
(item.value.created_at === existing.event.created_at && item.value.created_at > existing.event.created_at ||
item.addedAt > existing.addedAt)) { (item.value.created_at === existing.event.created_at && item.addedAt > existing.addedAt)
// This event is newer, mark the old one for deletion if it exists ) {
if (existing) { if (existing) {
keysToDelete.push(existing.key) keysToDelete.push(existing.key)
} }
eventMap.set(replaceableKey, { eventMap.set(dedupeKey, {
key: item.key, key: item.key,
event: item.value, event: item.value,
addedAt: item.addedAt addedAt: item.addedAt
}) })
} else { } else {
// This event is older or same, mark it for deletion
keysToDelete.push(item.key) keysToDelete.push(item.key)
} }
} catch (error) { } catch (error) {
// If we can't generate a replaceable key, skip this item logger.warn('Failed to get dedupe key for item', { key: item.key, error })
logger.warn('Failed to get replaceable key for item', { key: item.key, error })
invalidItemsCount++ invalidItemsCount++
continue
} }
} }
// Second pass: delete duplicates
const totalProcessed = eventMap.size + keysToDelete.length
const actualKept = eventMap.size const actualKept = eventMap.size
if (keysToDelete.length === 0) { if (keysToDelete.length === 0) {
// No duplicates found, but verify counts match const totalProcessed = eventMap.size + keysToDelete.length
if (totalProcessed + invalidItemsCount !== allItems.length) { if (totalProcessed + invalidItemsCount !== allItems.length) {
logger.warn('Count mismatch while cleaning up replaceable events', { logger.warn('Count mismatch while cleaning up replaceable events', {
totalItems: allItems.length, totalItems: allItems.length,
@ -2353,7 +2567,7 @@ class IndexedDbService {
invalid: invalidItemsCount invalid: invalidItemsCount
}) })
} }
return Promise.resolve({ deleted: 0, kept: actualKept }) return { deleted: 0, kept: actualKept }
} }
return new Promise((resolve) => { return new Promise((resolve) => {
@ -2363,14 +2577,13 @@ class IndexedDbService {
let deletedCount = 0 let deletedCount = 0
let completedCount = 0 let completedCount = 0
keysToDelete.forEach(key => { keysToDelete.forEach((key) => {
const deleteRequest = store.delete(key) const deleteRequest = store.delete(key)
deleteRequest.onsuccess = () => { deleteRequest.onsuccess = () => {
deletedCount++ deletedCount++
completedCount++ completedCount++
if (completedCount === keysToDelete.length) { if (completedCount === keysToDelete.length) {
transaction.commit() transaction.commit()
const actualKept = eventMap.size
const totalProcessed = actualKept + deletedCount const totalProcessed = actualKept + deletedCount
if (totalProcessed + invalidItemsCount !== allItems.length) { if (totalProcessed + invalidItemsCount !== allItems.length) {
logger.warn('Count mismatch after deletion', { logger.warn('Count mismatch after deletion', {
@ -2387,7 +2600,6 @@ class IndexedDbService {
completedCount++ completedCount++
if (completedCount === keysToDelete.length) { if (completedCount === keysToDelete.length) {
transaction.commit() transaction.commit()
const actualKept = eventMap.size
resolve({ deleted: deletedCount, kept: actualKept }) resolve({ deleted: deletedCount, kept: actualKept })
} }
} }
@ -3756,22 +3968,19 @@ class IndexedDbService {
} }
} }
private approxLibraryPublicationIndexBytes(ev: Event): number { private async listPublicationCatalogMasterRows(): Promise<
try { Array<{ key: string; lastAccessAt: number; bytes: number; hasNested: boolean }>
return new Blob([JSON.stringify(ev)]).size > {
} catch {
return 2048
}
}
async pruneUnverifiedLibraryPublicationIndexCacheEvents(): Promise<number> {
await this.initPromise await this.initPromise
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) return 0 if (!this.db?.objectStoreNames.contains(StoreNames.PUBLICATION_EVENTS)) return []
const nestedMasterKeys = new Set<string>()
const masters: Array<{ key: string; lastAccessAt: number; bytes: number }> = []
const toDelete: string[] = []
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.LIBRARY_PUBLICATION_INDEX, 'readonly') const tx = this.db!.transaction(StoreNames.PUBLICATION_EVENTS, 'readonly')
const req = tx.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX).openCursor() const store = tx.objectStore(StoreNames.PUBLICATION_EVENTS)
const req = store.openCursor()
req.onsuccess = () => { req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null const cursor = req.result as IDBCursorWithValue | null
if (!cursor) { if (!cursor) {
@ -3779,53 +3988,19 @@ class IndexedDbService {
resolve() resolve()
return return
} }
const row = cursor.value as TLibraryPublicationIndexCacheRow const row = cursor.value as TValue<Event>
if (row?.value?.kind === ExtendedKind.PUBLICATION && !isVerifiedPublicationIndex(row.value)) { const key = cursor.key as string
toDelete.push(cursor.key as string) if (row?.masterPublicationKey) {
} nestedMasterKeys.add(row.masterPublicationKey)
cursor.continue() } else if (
} row?.catalogMaster === 1 ||
req.onerror = (e) => { (row?.value?.kind === ExtendedKind.PUBLICATION && !row.masterPublicationKey)
tx.commit()
reject(idbEventToError(e))
}
})
for (const key of toDelete) {
await this.deleteStoreItem(StoreNames.LIBRARY_PUBLICATION_INDEX, key)
}
return toDelete.length
}
/** True when any row is keyed by event id instead of kind:pubkey:d address. */
async libraryPublicationIndexCacheHasLegacyKeys(): Promise<boolean> {
await this.initPromise
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) return false
return new Promise((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.LIBRARY_PUBLICATION_INDEX, 'readonly')
const req = tx.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX).openCursor()
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve(false)
return
}
const rowKey = cursor.key as string
const row = cursor.value as TLibraryPublicationIndexCacheRow
const ev = row?.value
const addr = ev ? eventTagAddress(ev) : null
if (
!ev ||
ev.kind !== ExtendedKind.PUBLICATION ||
!isStructuralPublicationIndex(ev) ||
!addr ||
rowKey !== addr
) { ) {
tx.commit() masters.push({
resolve(true) key,
return lastAccessAt: row.lastAccessAt ?? row.addedAt,
bytes: row.catalogBytes ?? approxPublicationCatalogMasterBytes(row.value as Event)
})
} }
cursor.continue() cursor.continue()
} }
@ -3834,241 +4009,94 @@ class IndexedDbService {
reject(idbEventToError(e)) reject(idbEventToError(e))
} }
}) })
}
async getLibraryPublicationIndexCacheEvents(): Promise<Event[]> { return masters.map((row) => ({ ...row, hasNested: nestedMasterKeys.has(row.key) }))
await this.initPromise
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) return []
return new Promise((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.LIBRARY_PUBLICATION_INDEX, 'readonly')
const req = tx.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX).openCursor()
const out: Event[] = []
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve(out)
return
}
const row = cursor.value as TLibraryPublicationIndexCacheRow
if (row?.value?.kind === ExtendedKind.PUBLICATION) out.push(row.value)
cursor.continue()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
})
} }
async getLibraryPublicationIndexCacheFootprint(): Promise<{ count: number; bytes: number }> { async getPublicationCatalogFootprint(): Promise<{ count: number; bytes: number }> {
await this.initPromise const rows = await this.listPublicationCatalogMasterRows()
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) { return {
return { count: 0, bytes: 0 } count: rows.length,
bytes: rows.reduce((sum, row) => sum + row.bytes, 0)
} }
return new Promise((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.LIBRARY_PUBLICATION_INDEX, 'readonly')
const req = tx.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX).openCursor()
let count = 0
let bytes = 0
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve({ count, bytes })
return
}
const row = cursor.value as TLibraryPublicationIndexCacheRow
count += 1
bytes += row.approxBytes ?? this.approxLibraryPublicationIndexBytes(row.value)
cursor.continue()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
})
} }
async mergeLibraryPublicationIndexCacheEvents( async mergePublicationCatalogIndexEvents(
events: Event[], events: Event[],
opts: { maxEntries: number; maxBytes: number } opts: { maxEntries: number; maxBytes: number }
): Promise<void> { ): Promise<void> {
await this.initPromise
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX) || events.length === 0) {
return
}
const now = Date.now()
const storeName = StoreNames.LIBRARY_PUBLICATION_INDEX
const rowsToWrite: Array<{ key: string; event: Event }> = []
for (const ev of events) { for (const ev of events) {
if (ev.kind !== ExtendedKind.PUBLICATION || !isStructuralPublicationIndex(ev)) continue if (ev.kind !== ExtendedKind.PUBLICATION || !isStructuralPublicationIndex(ev)) continue
const key = eventTagAddress(ev) await this.putReplaceableEvent(ev)
if (!key) continue
const existing = rowsToWrite.find((row) => row.key === key)
if (!existing) {
rowsToWrite.push({ key, event: ev })
continue
}
existing.event = pickNewerPublicationIndexEvent(existing.event, ev)
} }
await this.prunePublicationCatalogMasters(opts.maxEntries, opts.maxBytes)
}
if (rowsToWrite.length === 0) return async prunePublicationCatalogMasters(maxEntries: number, maxBytes: number): Promise<void> {
const rows = await this.listPublicationCatalogMasterRows()
const candidates = rows
.filter((r) => !r.hasNested)
.sort((a, b) => a.lastAccessAt - b.lastAccessAt)
let totalBytes = rows.reduce((s, r) => s + r.bytes, 0)
let totalCount = rows.length
const toDelete = new Set<string>()
await new Promise<void>((resolve, reject) => { for (const victim of candidates) {
const tx = this.db!.transaction(storeName, 'readwrite') if (totalCount <= maxEntries && totalBytes <= maxBytes) break
const store = tx.objectStore(storeName) toDelete.add(victim.key)
let pending = rowsToWrite.length totalBytes -= victim.bytes
totalCount -= 1
}
const finishOne = () => { for (const key of toDelete) {
pending -= 1 await this.deleteStoreItem(StoreNames.PUBLICATION_EVENTS, key)
if (pending === 0) { }
tx.commit() }
resolve()
}
}
for (const { key, event: ev } of rowsToWrite) { /** Remove relay-discovered catalog masters that were never opened (no nested sections cached). */
const get = store.get(key) async clearPublicationCatalogDiscoveryOnly(): Promise<void> {
get.onsuccess = () => { const rows = await this.listPublicationCatalogMasterRows()
const prev = get.result as TLibraryPublicationIndexCacheRow | undefined for (const row of rows) {
if (prev?.value && pickNewerPublicationIndexEvent(prev.value, ev).id === prev.value.id) { if (!row.hasNested) {
finishOne() await this.deleteStoreItem(StoreNames.PUBLICATION_EVENTS, row.key)
return
}
const row: TLibraryPublicationIndexCacheRow = {
key,
value: ev,
addedAt: prev?.addedAt ?? now,
lastAccessAt: now,
approxBytes: this.approxLibraryPublicationIndexBytes(ev)
}
const put = store.put(row)
put.onsuccess = () => finishOne()
put.onerror = (e) => {
finishOne()
if (pending === 0) reject(idbEventToError(e))
}
}
get.onerror = (e) => {
finishOne()
if (pending === 0) reject(idbEventToError(e))
}
} }
}) }
await this.pruneLibraryPublicationIndexCache(opts.maxEntries, opts.maxBytes)
} }
/** Drop invalid, legacy id-keyed, and superseded rows after an address-keyed merge. */ /** @deprecated Use {@link getPublicationCatalogFootprint}. */
async reconcileLibraryPublicationIndexCache(canonical: PublicationIndexMap): Promise<void> { async getLibraryPublicationIndexCacheFootprint(): Promise<{ count: number; bytes: number }> {
await this.initPromise return this.getPublicationCatalogFootprint()
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) return }
const toDelete: string[] = []
await new Promise<void>((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.LIBRARY_PUBLICATION_INDEX, 'readonly')
const req = tx.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX).openCursor()
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve()
return
}
const rowKey = cursor.key as string
const row = cursor.value as TLibraryPublicationIndexCacheRow
const ev = row?.value
const addr = ev ? eventTagAddress(ev) : null
const canon = addr ? canonical.get(addr) : undefined
const invalid =
!ev ||
ev.kind !== ExtendedKind.PUBLICATION ||
!isStructuralPublicationIndex(ev) ||
!addr ||
rowKey !== addr
const superseded = Boolean(canon && canon.id !== ev?.id)
if (invalid || superseded) {
toDelete.push(rowKey)
}
cursor.continue()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
})
for (const key of toDelete) { /** @deprecated Use {@link getPublicationCatalogIndexEvents}. */
await this.deleteStoreItem(StoreNames.LIBRARY_PUBLICATION_INDEX, key) async getLibraryPublicationIndexCacheEvents(): Promise<Event[]> {
} return this.getPublicationCatalogIndexEvents()
} }
async pruneLibraryPublicationIndexCache(maxEntries: number, maxBytes: number): Promise<void> { /** @deprecated Use {@link mergePublicationCatalogIndexEvents}. */
await this.initPromise async mergeLibraryPublicationIndexCacheEvents(
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) return events: Event[],
opts: { maxEntries: number; maxBytes: number }
): Promise<void> {
return this.mergePublicationCatalogIndexEvents(events, opts)
}
const rows: Array<{ key: string; lastAccessAt: number; bytes: number }> = [] /** @deprecated Use {@link clearPublicationCatalogDiscoveryOnly}. */
await new Promise<void>((resolve, reject) => { async clearLibraryPublicationIndexCacheStore(): Promise<void> {
const tx = this.db!.transaction(StoreNames.LIBRARY_PUBLICATION_INDEX, 'readonly') return this.clearPublicationCatalogDiscoveryOnly()
const req = tx.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX).openCursor() }
req.onsuccess = () => {
const cursor = req.result as IDBCursorWithValue | null
if (!cursor) {
tx.commit()
resolve()
return
}
const row = cursor.value as TLibraryPublicationIndexCacheRow
rows.push({
key: cursor.key as string,
lastAccessAt: row.lastAccessAt ?? row.addedAt,
bytes: row.approxBytes ?? this.approxLibraryPublicationIndexBytes(row.value)
})
cursor.continue()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
})
rows.sort((a, b) => a.lastAccessAt - b.lastAccessAt) /** @deprecated No-op after v43 consolidation. */
const toDelete = new Set<string>() async reconcileLibraryPublicationIndexCache(_canonical: PublicationIndexMap): Promise<void> {}
let totalBytes = rows.reduce((s, r) => s + r.bytes, 0)
let totalCount = rows.length
while (totalCount > maxEntries || totalBytes > maxBytes) {
const victim = rows.shift()
if (!victim) break
toDelete.add(victim.key)
totalBytes -= victim.bytes
totalCount -= 1
}
for (const key of toDelete) { /** @deprecated No-op after v43 consolidation. */
await this.deleteStoreItem(StoreNames.LIBRARY_PUBLICATION_INDEX, key) async libraryPublicationIndexCacheHasLegacyKeys(): Promise<boolean> {
} return false
} }
async clearLibraryPublicationIndexCacheStore(): Promise<void> { /** @deprecated No-op after v43 consolidation. */
await this.initPromise async pruneUnverifiedLibraryPublicationIndexCacheEvents(): Promise<number> {
if (!this.db?.objectStoreNames.contains(StoreNames.LIBRARY_PUBLICATION_INDEX)) return return 0
await new Promise<void>((resolve, reject) => {
const tx = this.db!.transaction(StoreNames.LIBRARY_PUBLICATION_INDEX, 'readwrite')
const req = tx.objectStore(StoreNames.LIBRARY_PUBLICATION_INDEX).clear()
req.onsuccess = () => {
tx.commit()
resolve()
}
req.onerror = (e) => {
tx.commit()
reject(idbEventToError(e))
}
})
} }
/** /**

Loading…
Cancel
Save