diff --git a/src/components/Note/PublicationCoverFallback.tsx b/src/components/Note/PublicationCoverFallback.tsx index 913679f6..b934daf6 100644 --- a/src/components/Note/PublicationCoverFallback.tsx +++ b/src/components/Note/PublicationCoverFallback.tsx @@ -17,7 +17,9 @@ export default function PublicationCoverFallback({ const isLibrary = size === 'library' const maxClass = isLibrary ? LIBRARY_PUBLICATION_COVER_MAX_CLASS : PUBLICATION_COVER_MAX_CLASS - const stackedLayoutClass = isLibrary ? 'aspect-[3/4] w-full' : 'aspect-[3/4] w-48 max-w-full' + const stackedLayoutClass = isLibrary + ? 'h-[200px] w-[200px] max-h-[200px] max-w-[200px]' + : 'aspect-[3/4] w-48 max-w-full' return (
} - const stackedLayoutClass = isLibrary ? 'aspect-[3/4] w-full' : 'w-fit' + const stackedLayoutClass = isLibrary ? 'w-fit max-w-full' : 'w-fit' // Library grid: always load covers (user opened Bibliothek). Tap-to-reveal on the card would // fight PublicationCard navigation, leaving blurhash placeholders stuck forever. @@ -62,11 +62,12 @@ export default function PublicationCoverImage({ return (
e.stopPropagation() : undefined} @@ -74,8 +75,8 @@ export default function PublicationCoverImage({ Array.from( new Set([ + ...LIBRARY_RELAY_URLS.map((url) => normalizeAnyRelayUrl(url) || url), + ...DOCUMENT_RELAY_URLS.map((url) => normalizeAnyRelayUrl(url) || url), ...currentBrowsingRelayUrls.map((url) => normalizeAnyRelayUrl(url) || url), ...favoriteRelays.map((url) => normalizeAnyRelayUrl(url) || url), ...FAST_READ_RELAY_URLS.map((url) => normalizeAnyRelayUrl(url) || url) diff --git a/src/constants.ts b/src/constants.ts index 52e1b8ca..4ee67636 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -463,6 +463,7 @@ export const BOOKSTR_RELAY_URLS = [ */ export const DOCUMENT_RELAY_URLS = [ 'wss://thecitadel.nostr1.com', + 'wss://theforest.nostr1.com', 'wss://relay.wikifreedia.xyz', 'wss://essayist.decentnewsroom.com' ] as const diff --git a/src/hooks/useLibraryPublications.ts b/src/hooks/useLibraryPublications.ts index 4edd69e4..e1c95434 100644 --- a/src/hooks/useLibraryPublications.ts +++ b/src/hooks/useLibraryPublications.ts @@ -72,8 +72,9 @@ export function useLibraryPublications(isActive: boolean) { const [pinListEvent, setPinListEvent] = useState(null) const [myBooklistTargets, setMyBooklistTargets] = useState(EMPTY_BOOKLIST_TARGETS) const [booklistTargetsLoading, setBooklistTargetsLoading] = useState(false) - const loadGenRef = useRef(0) - const indexesReadyGenRef = useRef(0) + const [reloadNonce, setReloadNonce] = useState(0) + const forceRefreshNextLoadRef = useRef(false) + const indexesReadyRef = useRef(false) const [mineIndexEntries, setMineIndexEntries] = useState([]) const [mineFilterComputing, setMineFilterComputing] = useState(false) const mineIndexCacheRef = useRef<{ @@ -147,69 +148,95 @@ export function useLibraryPublications(isActive: boolean) { [] ) - const load = useCallback( - async (forceRefresh = false) => { - const gen = ++loadGenRef.current - setLoading(true) - setEngagementLoading(false) - setError(null) - setFeedPageIndex(0) - if (import.meta.env.DEV) { - logger.info('[Library] page load requested', { forceRefresh, gen }) - } + const applyIndexesSnapshot = useCallback( + ( + snapshot: { + indexEvents: Event[] + allIndexCount: number + topLevelCount: number + }, + engagementMaps: PublicationEngagementMaps, + pageIndex: number + ) => { + setIndexEvents(snapshot.indexEvents) + setAllIndexCount(snapshot.allIndexCount) + setTopLevelCount(snapshot.topLevelCount) + applyDefaultFeedSlice(snapshot.indexEvents, engagementMaps, pageIndex) + }, + [applyDefaultFeedSlice] + ) + + useEffect(() => { + if (!isActive) return + let cancelled = false + indexesReadyRef.current = false + setLoading(true) + setEngagementLoading(false) + setError(null) + setFeedPageIndex(0) + const forceRefresh = forceRefreshNextLoadRef.current + forceRefreshNextLoadRef.current = false + if (import.meta.env.DEV) { + logger.info('[Library] page load requested', { forceRefresh, reloadNonce }) + } + + void (async () => { try { const relays = await buildLibraryRelayUrls(pubkey || undefined, blockedRelays ?? []) - indexesReadyGenRef.current = 0 + if (cancelled) return const result = await loadLibraryPublicationIndex(relays, { forceRefresh, viewerPubkey: pubkey || undefined, onIndexesReady: (snapshot) => { - if (gen !== loadGenRef.current) return - indexesReadyGenRef.current = gen - setIndexEvents(snapshot.indexEvents) - setAllIndexCount(snapshot.allIndexCount) - setTopLevelCount(snapshot.topLevelCount) - applyDefaultFeedSlice(snapshot.indexEvents, EMPTY_ENGAGEMENT, 0) + if (cancelled) return + indexesReadyRef.current = true + if (import.meta.env.DEV && snapshot.indexEvents.length > 0) { + logger.info('[Library] indexes ready (progress)', { + validCount: snapshot.indexEvents.length, + topLevelCount: snapshot.topLevelCount, + entryCount: snapshot.engaged.length + }) + } + applyIndexesSnapshot(snapshot, EMPTY_ENGAGEMENT, 0) setLoading(false) setEngagementLoading(true) } }) - if (gen !== loadGenRef.current) return - setIndexEvents(result.indexEvents) + if (cancelled) return + applyIndexesSnapshot(result, result.engagement, 0) setEngagement(result.engagement) - setAllIndexCount(result.allIndexCount) - setTopLevelCount(result.topLevelCount) - applyDefaultFeedSlice(result.indexEvents, result.engagement, 0) } catch (e) { - if (gen !== loadGenRef.current) return - if (indexesReadyGenRef.current === gen) { + if (cancelled) return + if (indexesReadyRef.current) { if (import.meta.env.DEV) { logger.warn('[Library] engagement phase failed after indexes loaded', { - message: e instanceof Error ? e.message : String(e), - gen + message: e instanceof Error ? e.message : String(e) }) } } else { const message = e instanceof Error ? e.message : 'Failed to load library' setError(message) if (import.meta.env.DEV) { - logger.warn('[Library] page load failed', { message, gen }) + logger.warn('[Library] page load failed', { message }) } } } finally { - if (gen === loadGenRef.current) { + if (!cancelled) { setLoading(false) setEngagementLoading(false) } } - }, - [pubkey, blockedRelays, applyDefaultFeedSlice] - ) + })() - useEffect(() => { - if (!isActive) return - void load(false) - }, [isActive, load]) + return () => { + cancelled = true + } + }, [isActive, pubkey, blockedRelays, reloadNonce, applyIndexesSnapshot]) + + const refresh = useCallback(() => { + forceRefreshNextLoadRef.current = true + void clearAllLibraryIndexCaches().then(() => setReloadNonce((n) => n + 1)) + }, []) useEffect(() => { if (!isActive || !pubkey || indexEvents.length === 0) return @@ -266,10 +293,6 @@ export function useLibraryPublications(isActive: boolean) { } }, [debouncedSearch, indexEvents, engagement]) - const refresh = useCallback(() => { - void clearAllLibraryIndexCaches().then(() => load(true)) - }, [load]) - const searchOnRelays = useCallback(async () => { const q = searchQuery.trim() if (!q) return diff --git a/src/lib/library-index-idb-cache.ts b/src/lib/library-index-idb-cache.ts index a5df2839..ab12f8e5 100644 --- a/src/lib/library-index-idb-cache.ts +++ b/src/lib/library-index-idb-cache.ts @@ -4,20 +4,21 @@ import { getLibraryIndexCacheBudget } from '@/lib/library-index-cache-config' import logger from '@/lib/logger' -import { isVerifiedPublicationIndex } from '@/lib/publication-index' +import { filterStructuralIndexEvents } from '@/lib/publication-index' import indexedDb from '@/services/indexed-db.service' import type { Event } from 'nostr-tools' export async function loadLibraryIndexCacheEvents(): Promise { try { const cached = await indexedDb.getLibraryPublicationIndexCacheEvents() - const verified = cached.filter(isVerifiedPublicationIndex) - if (verified.length < cached.length) { + // IDB rows were verified on write; structural re-check only (avoid ~5k verifyEvent on read). + const structural = filterStructuralIndexEvents(cached) + if (structural.length < cached.length) { void indexedDb .pruneUnverifiedLibraryPublicationIndexCacheEvents() .catch(() => {}) } - return verified + return structural } catch (e) { if (import.meta.env.DEV) { logger.warn('[Library] index IDB read failed', { diff --git a/src/lib/library-publication-index.ts b/src/lib/library-publication-index.ts index abcc3bc6..dd6d02bf 100644 --- a/src/lib/library-publication-index.ts +++ b/src/lib/library-publication-index.ts @@ -23,6 +23,7 @@ import { getReplaceableCoordinateFromEvent, isReplaceableEvent } from '@/lib/eve import { verifyEvent } from 'nostr-tools' import { isEventInPinList } from '@/lib/replaceable-list-latest' import { isRelayBlockedByUser } from '@/lib/relay-blocked' +import { stripLocalNetworkRelaysForWssReq } from '@/lib/relay-list-sanitize' import { buildComprehensiveRelayList } from '@/lib/relay-list-builder' import { clearLibraryIndexIdbCache, @@ -41,9 +42,12 @@ import { queryService } from '@/services/client.service' import type { Event, Filter } from 'nostr-tools' import { kinds, nip19 } from 'nostr-tools' -const INDEX_FETCH_LIMIT = 500 +const INDEX_WS_PAGE_LIMIT = 500 const INDEX_HTTP_PAGE_LIMIT = 100 -const INDEX_HTTP_MAX_PAGES = 5 +/** Cursor pages per relay for library index bulk load (up to ~50k WS / ~10k HTTP rows each). */ +const INDEX_MAX_PAGES_PER_RELAY = 100 +/** verifyEvent batch size — yield between chunks so the main thread stays responsive. */ +const INDEX_VERIFY_CHUNK = 80 const ENGAGEMENT_ADDRESS_CHUNK = 36 const ENGAGEMENT_EVENT_ID_CHUNK = 44 const MAX_TARGET_ADDRESSES = 480 @@ -56,11 +60,21 @@ export const LIBRARY_RELAY_SEARCH_LIMIT = 100 const LIBRARY_RELAY_SEARCH_TIMEOUT_MS = 28_000 /** NIP-51 pin list (kind 10001). */ const PIN_LIST_KIND = 10001 -const QUERY_OPTS = { - globalTimeout: 18_000, - eoseTimeout: 3_000, - firstRelayResultGraceMs: false as const -} +/** Per-relay WS page fetch — one relay at a time avoids multi-relay onclose resolving after ~1s. */ +const LIBRARY_INDEX_QUERY_OPTS = { + globalTimeout: 45_000, + eoseTimeout: 8_000, + firstRelayResultGraceMs: false as const, + foreground: true +} as const + +/** First-page batch: unblock the library grid quickly. */ +const LIBRARY_INDEX_FIRST_PAGE_OPTS = { + globalTimeout: 12_000, + eoseTimeout: 5_000, + firstRelayResultGraceMs: false as const, + foreground: true +} as const const ENGAGEMENT_QUERY_OPTS = { globalTimeout: 45_000, @@ -117,6 +131,56 @@ type LibraryIndexCache = { let sessionCache: LibraryIndexCache | null = null +type LibraryIndexLoadSnapshot = { + engaged: LibraryPublicationEntry[] + allIndexCount: number + topLevelCount: number + indexEvents: Event[] +} + +type LibraryIndexLoadResult = LibraryIndexLoadSnapshot & { + engagement: PublicationEngagementMaps +} + +type LibraryIndexLoadJob = { + relayKey: string + forceRefresh: boolean + promise: Promise + onIndexesReadyListeners: Array<(snapshot: LibraryIndexLoadSnapshot) => void> + lastProgressEvents: Event[] | null +} + +let indexLoadJob: LibraryIndexLoadJob | null = null + +function emitIndexesReadySnapshot( + listeners: Array<(snapshot: LibraryIndexLoadSnapshot) => void>, + indexEvents: Event[] +) { + if (listeners.length === 0) return + const indexByAddress = buildIndexByAddress(indexEvents) + const topLevel = getTopLevelIndexEvents(indexEvents) + const snapshot: LibraryIndexLoadSnapshot = { + engaged: buildRecentPublicationEntries(topLevel, indexByAddress, emptyPublicationEngagementMaps()), + allIndexCount: indexEvents.length, + topLevelCount: topLevel.length, + indexEvents + } + for (const listener of listeners) { + listener(snapshot) + } +} + +function registerIndexesReadyListener( + job: LibraryIndexLoadJob, + listener?: (snapshot: LibraryIndexLoadSnapshot) => void +) { + if (!listener) return + job.onIndexesReadyListeners.push(listener) + if (job.lastProgressEvents) { + emitIndexesReadySnapshot([listener], job.lastProgressEvents) + } +} + type LibrarySearchSessionRow = { fingerprint: string entries: LibraryPublicationEntry[] @@ -238,16 +302,120 @@ function chunkArray(items: T[], size: number): T[][] { return out } -async function fetchPaginatedFromHttpIndexRelay(baseUrl: string, filter: Filter): Promise { +function isLibraryDeepIndexRelay(url: string): boolean { + const key = canonicalRelaySessionKey(normalizeLibraryRelayUrl(url) || url) + return key !== '' && LIBRARY_DEEP_INDEX_RELAY_KEYS.has(key) +} + +function oldestCreatedAt(events: Event[]): number { + let oldest = Number.MAX_SAFE_INTEGER + for (const ev of events) { + if (ev.created_at < oldest) oldest = ev.created_at + } + return oldest +} + +function mergeIndexPageBatch(out: Event[], seen: Set, batch: Event[]): number { + let oldest = batch[0]?.created_at ?? Number.MAX_SAFE_INTEGER + for (const ev of batch) { + if (ev.created_at < oldest) oldest = ev.created_at + if (seen.has(ev.id)) continue + seen.add(ev.id) + out.push(ev) + } + return oldest +} + +async function fetchWsIndexFirstPage(wsRelay: string, filter: Filter): Promise { + const pageFilter: Filter = { ...filter, limit: INDEX_WS_PAGE_LIMIT } + try { + return await queryService.fetchEvents([wsRelay], [pageFilter], LIBRARY_INDEX_FIRST_PAGE_OPTS) + } catch (e) { + if (import.meta.env.DEV) { + logger.warn('[Library] WS index first page failed', { + wsRelay, + message: e instanceof Error ? e.message : String(e) + }) + } + return [] + } +} + +async function fetchHttpIndexFirstPage(baseUrl: string, filter: Filter): Promise { + const pageFilter: Filter = { ...filter, limit: INDEX_HTTP_PAGE_LIMIT } + try { + const pageResult = await queryIndexRelayForLibrary(baseUrl, pageFilter) + return pageResult.events + } catch (e) { + if (import.meta.env.DEV) { + logger.warn('[Library] HTTP index first page failed', { + baseUrl, + message: e instanceof Error ? e.message : String(e) + }) + } + return [] + } +} + +async function fetchRemainingPagesFromWsIndexRelay( + wsRelay: string, + filter: Filter, + firstPage: Event[] +): Promise { + if (firstPage.length < INDEX_WS_PAGE_LIMIT) return [] + const out: Event[] = [] - const seen = new Set() - let until: number | undefined + const seen = new Set(firstPage.map((ev) => ev.id)) + let until = oldestCreatedAt(firstPage) - 1 + if (until < 0) return [] - for (let page = 0; page < INDEX_HTTP_MAX_PAGES; page++) { + for (let page = 1; page < INDEX_MAX_PAGES_PER_RELAY; page++) { + const pageFilter: Filter = { + ...filter, + limit: INDEX_WS_PAGE_LIMIT, + until + } + let batch: Event[] = [] + try { + batch = await queryService.fetchEvents([wsRelay], [pageFilter], LIBRARY_INDEX_QUERY_OPTS) + } catch (e) { + if (import.meta.env.DEV) { + logger.warn('[Library] WS index page failed', { + wsRelay, + page, + message: e instanceof Error ? e.message : String(e) + }) + } + break + } + if (batch.length === 0) break + + const oldest = mergeIndexPageBatch(out, seen, batch) + if (batch.length < INDEX_WS_PAGE_LIMIT) break + if (oldest === Number.MAX_SAFE_INTEGER) break + until = oldest - 1 + } + + return out +} + +async function fetchRemainingPagesFromHttpIndexRelay( + baseUrl: string, + filter: Filter, + firstPage: Event[] +): Promise { + if (firstPage.length === 0) return [] + + const out: Event[] = [] + const seen = new Set(firstPage.map((ev) => ev.id)) + let until = oldestCreatedAt(firstPage) - 1 + if (until < 0) return [] + + for (let page = 1; page < INDEX_MAX_PAGES_PER_RELAY; page++) { const pageFilter: Filter = { ...filter, limit: INDEX_HTTP_PAGE_LIMIT, - ...(until != null ? { until } : {}) + until } let batch: Event[] = [] let apiRowCount = 0 @@ -267,14 +435,7 @@ async function fetchPaginatedFromHttpIndexRelay(baseUrl: string, filter: Filter) } if (apiRowCount === 0) break - let oldest = batch[0]?.created_at ?? Number.MAX_SAFE_INTEGER - for (const ev of batch) { - if (ev.created_at < oldest) oldest = ev.created_at - if (seen.has(ev.id)) continue - seen.add(ev.id) - out.push(ev) - } - + const oldest = mergeIndexPageBatch(out, seen, batch) if (apiRowCount < INDEX_HTTP_PAGE_LIMIT) break if (oldest === Number.MAX_SAFE_INTEGER) break until = oldest - 1 @@ -291,6 +452,12 @@ function normalizeLibraryRelayUrl(url: string): string { return normalizeUrl(trimmed) || trimmed } +const LIBRARY_DEEP_INDEX_RELAY_KEYS = new Set( + LIBRARY_RELAY_URLS.map((url) => + canonicalRelaySessionKey(normalizeLibraryRelayUrl(url) || url) + ).filter(Boolean) +) + function filterBlockedLibraryRelays(urls: string[], blockedRelays: readonly string[] = []): string[] { if (blockedRelays.length === 0) return urls return urls.filter((url) => !isRelayBlockedByUser(url, blockedRelays)) @@ -379,50 +546,147 @@ function engagementMapsSizeSummary(maps: PublicationEngagementMaps): Record { +export type FetchLibraryIndexEventsOptions = { + /** Called when IDB cache and each network batch are ready — unblocks the library grid early. */ + onProgress?: (events: Event[]) => void +} + +async function filterValidNewIndexEvents( + incoming: Event[], + knownIds: ReadonlySet +): Promise { + const novel = incoming.filter((event) => !knownIds.has(event.id)) + if (novel.length === 0) return [] + const out: Event[] = [] + for (let i = 0; i < novel.length; i += INDEX_VERIFY_CHUNK) { + out.push(...filterValidIndexEvents(novel.slice(i, i + INDEX_VERIFY_CHUNK))) + if (i + INDEX_VERIFY_CHUNK < novel.length) { + await new Promise((resolve) => { + setTimeout(resolve, 0) + }) + } + } + return out +} + +async function mergeValidIndexBatch( + existing: Event[], + knownIds: Set, + incoming: Event[] +): Promise { + const newValid = await filterValidNewIndexEvents(incoming, knownIds) + if (newValid.length === 0) return existing + for (const event of newValid) knownIds.add(event.id) + return dedupeEventsById([...existing, ...newValid]) +} + +export async function fetchLibraryIndexEvents( + relayUrls: string[], + options?: FetchLibraryIndexEventsOptions +): Promise { const indexRelays = libraryIndexRelayUrls(relayUrls) if (indexRelays.length === 0) return [] const cached = await loadLibraryIndexCacheEvents() - const filter: Filter = { kinds: [ExtendedKind.PUBLICATION], limit: INDEX_FETCH_LIMIT } - const { wsRelays, httpRelays } = splitWsAndHttpRelays(indexRelays) + let validMerged = dedupeEventsById(cached) + const knownValidIds = new Set(validMerged.map((event) => event.id)) + const emitProgress = () => { + options?.onProgress?.(validMerged) + } + emitProgress() + + const filter: Filter = { kinds: [ExtendedKind.PUBLICATION], limit: INDEX_WS_PAGE_LIMIT } + const { wsRelays: rawWsRelays, httpRelays } = splitWsAndHttpRelays(indexRelays) + const wsRelays = stripLocalNetworkRelaysForWssReq(rawWsRelays) + + const firstPageByRelay = new Map() + const firstPagePromises: Promise<{ relay: string; events: Event[] }>[] = [ + ...wsRelays.map(async (wsRelay) => { + const events = await fetchWsIndexFirstPage(wsRelay, filter) + return { relay: wsRelay, events } + }), + ...httpRelays.map(async (httpRelay) => { + const events = await fetchHttpIndexFirstPage(httpRelay, filter) + return { relay: httpRelay, events } + }) + ] - const batches: Promise[] = [] - if (wsRelays.length > 0) { - batches.push( - queryService.fetchEvents(wsRelays, [filter], QUERY_OPTS).catch((e) => { - if (import.meta.env.DEV) { - logger.warn('[Library] WS index fetch failed', { - message: e instanceof Error ? e.message : String(e) - }) - } - return [] as Event[] - }) - ) - } - for (const httpRelay of httpRelays) { - batches.push(fetchPaginatedFromHttpIndexRelay(httpRelay, filter)) + const firstSettled = await Promise.allSettled(firstPagePromises) + for (const result of firstSettled) { + if (result.status !== 'fulfilled') continue + firstPageByRelay.set(result.value.relay, result.value.events) } - const settled = await Promise.allSettled(batches) - const networkMerged = dedupeEventsById( - settled.flatMap((r) => (r.status === 'fulfilled' ? r.value : [])) + const firstPageNetwork = dedupeEventsById( + firstSettled.flatMap((r) => (r.status === 'fulfilled' ? r.value.events : [])) ) - const merged = dedupeEventsById([...cached, ...networkMerged]) - const valid = filterValidIndexEvents(merged) - void persistLibraryIndexCacheEvents(valid) + validMerged = await mergeValidIndexBatch(validMerged, knownValidIds, firstPageNetwork) + void persistLibraryIndexCacheEvents(validMerged) + emitProgress() + if (import.meta.env.DEV) { - logger.info('[Library] index fetch', { + const perRelayFirstPageCounts = firstSettled.map((r) => + r.status === 'fulfilled' + ? { relay: r.value.relay, count: r.value.events.length } + : { relay: 'unknown', count: 0 } + ) + logger.info('[Library] index first page', { indexRelays: indexRelays.length, wsRelays: wsRelays.length, httpRelays: httpRelays.length, + strippedWsRelays: rawWsRelays.length - wsRelays.length, cachedCount: cached.length, - networkCount: networkMerged.length, - mergedCount: merged.length, - validCount: valid.length + firstPageCount: firstPageNetwork.length, + mergedCount: validMerged.length, + validCount: validMerged.length, + topLevelCount: getTopLevelIndexEvents(validMerged).length, + perRelayCounts: perRelayFirstPageCounts }) } - return valid + + const deepBatches: Promise<{ relay: string; events: Event[] }>[] = [] + for (const wsRelay of wsRelays) { + if (!isLibraryDeepIndexRelay(wsRelay)) continue + deepBatches.push( + fetchRemainingPagesFromWsIndexRelay(wsRelay, filter, firstPageByRelay.get(wsRelay) ?? []).then( + (events) => ({ relay: wsRelay, events }) + ) + ) + } + for (const httpRelay of httpRelays) { + if (!isLibraryDeepIndexRelay(httpRelay)) continue + deepBatches.push( + fetchRemainingPagesFromHttpIndexRelay( + httpRelay, + filter, + firstPageByRelay.get(httpRelay) ?? [] + ).then((events) => ({ relay: httpRelay, events })) + ) + } + + const deepSettled = await Promise.allSettled(deepBatches) + const deepNetwork = dedupeEventsById( + deepSettled.flatMap((r) => (r.status === 'fulfilled' ? r.value.events : [])) + ) + validMerged = await mergeValidIndexBatch(validMerged, knownValidIds, deepNetwork) + void persistLibraryIndexCacheEvents(validMerged) + emitProgress() + + if (import.meta.env.DEV) { + const perRelayDeepCounts = deepSettled.map((r) => + r.status === 'fulfilled' + ? { relay: r.value.relay, count: r.value.events.length } + : { relay: 'unknown', count: 0 } + ) + logger.info('[Library] index fetch complete', { + deepPageCount: deepNetwork.length, + mergedCount: validMerged.length, + validCount: validMerged.length, + topLevelCount: getTopLevelIndexEvents(validMerged).length, + perRelayDeepCounts + }) + } + return validMerged } export function buildEngagementMapsFromEvents( @@ -1749,26 +2013,58 @@ export async function loadLibraryPublicationIndex( forceRefresh?: boolean viewerPubkey?: string | null /** Called as soon as kind-30040 indexes are loaded — before engagement (which can take minutes). */ - onIndexesReady?: (snapshot: { - engaged: LibraryPublicationEntry[] - allIndexCount: number - topLevelCount: number - indexEvents: Event[] - }) => void + onIndexesReady?: (snapshot: LibraryIndexLoadSnapshot) => void } -): Promise<{ - engaged: LibraryPublicationEntry[] - allIndexCount: number - topLevelCount: number - indexEvents: Event[] - engagement: PublicationEngagementMaps -}> { - const key = relaySetKey(relayUrls) +): Promise { + const relayKey = relaySetKey(relayUrls) + const forceRefresh = options?.forceRefresh ?? false + + if (!forceRefresh && indexLoadJob?.relayKey === relayKey && !indexLoadJob.forceRefresh) { + registerIndexesReadyListener(indexLoadJob, options?.onIndexesReady) + if (import.meta.env.DEV) { + logger.info('[Library] load joined in-flight', { + relayCount: relayUrls.length, + hasProgress: indexLoadJob.lastProgressEvents != null + }) + } + return indexLoadJob.promise + } + + const job: LibraryIndexLoadJob = { + relayKey, + forceRefresh, + onIndexesReadyListeners: [], + lastProgressEvents: null, + promise: Promise.resolve(null as unknown as LibraryIndexLoadResult) + } + registerIndexesReadyListener(job, options?.onIndexesReady) + indexLoadJob = job + job.promise = runLibraryPublicationIndexLoad(relayUrls, options, job).finally(() => { + if (indexLoadJob === job) indexLoadJob = null + }) + return job.promise +} + +async function runLibraryPublicationIndexLoad( + relayUrls: string[], + options: { + forceRefresh?: boolean + viewerPubkey?: string | null + onIndexesReady?: (snapshot: LibraryIndexLoadSnapshot) => void + } | undefined, + job: LibraryIndexLoadJob +): Promise { + const key = job.relayKey const viewerPubkey = options?.viewerPubkey ?? null if (import.meta.env.DEV) { logger.info('[Library] load start', { relayCount: relayUrls.length, cached: sessionCache?.relayKey === key }) } + const emitIndexesReady = (indexEvents: Event[]) => { + job.lastProgressEvents = indexEvents + emitIndexesReadySnapshot(job.onIndexesReadyListeners, indexEvents) + } + if (!options?.forceRefresh && sessionCache?.relayKey === key) { if (sessionCache.viewerPubkey !== viewerPubkey) { const targetAddresses = collectTargetAddressesFromIndexes( @@ -1802,6 +2098,7 @@ export async function loadLibraryPublicationIndex( if (import.meta.env.DEV) { logger.info('[Library] load from cache', { engaged: engaged.length }) } + emitIndexesReady(sessionCache.indexEvents) return { engaged, allIndexCount: sessionCache.indexEvents.length, @@ -1811,7 +2108,9 @@ export async function loadLibraryPublicationIndex( } } - const indexEvents = await fetchLibraryIndexEvents(relayUrls) + const indexEvents = await fetchLibraryIndexEvents(relayUrls, { + onProgress: emitIndexesReady + }) if (import.meta.env.DEV) { logger.info('[Library] indexes fetched', { validCount: indexEvents.length }) } @@ -1819,12 +2118,7 @@ export async function loadLibraryPublicationIndex( const indexByAddress = buildIndexByAddress(indexEvents) let topLevel = getTopLevelIndexEvents(indexEvents) - options?.onIndexesReady?.({ - engaged: buildRecentPublicationEntries(topLevel, indexByAddress, emptyPublicationEngagementMaps()), - allIndexCount: indexEvents.length, - topLevelCount: topLevel.length, - indexEvents - }) + emitIndexesReady(indexEvents) const topLevelForHydrate = topLevel await hydrateNestedIndexEvents(indexEvents, indexByAddress, relayUrls, { @@ -1895,12 +2189,14 @@ export async function loadLibraryPublicationIndex( export function clearLibraryPublicationIndexCache(): void { sessionCache = null + indexLoadJob = null clearLibrarySearchSessionCache() } /** Clears Library tab session + IDB index cache only (publication reading cache is unchanged). */ export async function clearAllLibraryIndexCaches(): Promise { sessionCache = null + indexLoadJob = null clearLibrarySearchSessionCache() await clearLibraryIndexIdbCache() } diff --git a/src/lib/publication-asciidoc-assembler.ts b/src/lib/publication-asciidoc-assembler.ts index f6651fd4..607f08b6 100644 --- a/src/lib/publication-asciidoc-assembler.ts +++ b/src/lib/publication-asciidoc-assembler.ts @@ -10,6 +10,7 @@ import { publicationRefKey, type PublicationSectionRef } from '@/lib/publication-section-fetch' +import { uppercaseRomanNumeralsInText } from '@/lib/roman-numeral-display' import type { Event } from 'nostr-tools' const MAX_NEST_DEPTH = 8 @@ -40,7 +41,8 @@ function tagValue(event: Event, name: string): string | undefined { } function titleFromIndex(event: Event): string { - return tagValue(event, 'title') || tagValue(event, 'd') || 'Publication' + const raw = tagValue(event, 'title') || tagValue(event, 'd') || 'Publication' + return uppercaseRomanNumeralsInText(raw) } function authorFromMetadata(metadata: PublicationIndexMetadata, pubkey: string): string { @@ -121,7 +123,7 @@ function appendIndexBody( if (!sectionTitle && ref.coordinate) { sectionTitle = ref.coordinate.split(':').slice(2).join(':') } - if (sectionTitle) parts.push(heading(headingLevel, sectionTitle)) + if (sectionTitle) parts.push(heading(headingLevel, uppercaseRomanNumeralsInText(sectionTitle))) const body = article.content.trim() if (body) parts.push(`${body}\n\n`) @@ -129,7 +131,9 @@ function appendIndexBody( } else if (ref.type === 'e') { const article = resolveRefEvent(ref, fetched) if (!article) continue - const sectionTitle = tagValue(article, 'title')?.trim() || 'Section' + const sectionTitle = uppercaseRomanNumeralsInText( + tagValue(article, 'title')?.trim() || 'Section' + ) parts.push(heading(headingLevel, sectionTitle)) const body = article.content.trim() if (body) parts.push(`${body}\n\n`) diff --git a/src/lib/publication-index.ts b/src/lib/publication-index.ts index 94f5d4a0..e4928c33 100644 --- a/src/lib/publication-index.ts +++ b/src/lib/publication-index.ts @@ -29,19 +29,28 @@ export function eventTagAddress(event: Event): string | null { return `${event.kind}:${event.pubkey.toLowerCase()}:${d}` } -/** Removes kind 30040 index events that don't comply with NKBIP-01. */ +/** NKBIP-01 shape checks only — no signature verification (cheap for large IDB reads). */ +export function isStructuralPublicationIndex(event: Event): boolean { + if (event.kind !== ExtendedKind.PUBLICATION) return false + if ((event.content ?? '') !== '') return false + const hasTitle = event.tags.some( + (t) => (t[0] || '').trim().toLowerCase() === 'title' && t[1] + ) + const hasD = event.tags.some((t) => (t[0] || '').trim().toLowerCase() === 'd' && t[1]) + const hasA = event.tags.some((t) => t[0] === 'a' && t[1]) + const hasE = event.tags.some((t) => t[0] === 'e' && t[1]) + return hasTitle && hasD && (hasA || hasE) +} + +export function filterStructuralIndexEvents(events: Event[]): Event[] { + return events.filter(isStructuralPublicationIndex) +} + +/** Removes kind 30040 index events that don't comply with NKBIP-01 (includes signature check). */ export function filterValidIndexEvents(events: Event[]): Event[] { - return events.filter((event) => { - if (event.kind !== ExtendedKind.PUBLICATION) return false - if ((event.content ?? '') !== '') return false - const hasTitle = event.tags.some( - (t) => (t[0] || '').trim().toLowerCase() === 'title' && t[1] - ) - const hasD = event.tags.some((t) => (t[0] || '').trim().toLowerCase() === 'd' && t[1]) - const hasA = event.tags.some((t) => t[0] === 'a' && t[1]) - const hasE = event.tags.some((t) => t[0] === 'e' && t[1]) - return hasTitle && hasD && (hasA || hasE) && isVerifiedPublicationIndex(event) - }) + return events.filter( + (event) => isStructuralPublicationIndex(event) && isVerifiedPublicationIndex(event) + ) } export function collectPublicationATagRefs(event: Event): PublicationSectionRef[] { diff --git a/src/lib/publication-section-fetch.ts b/src/lib/publication-section-fetch.ts index 4e8d18f5..0f599169 100644 --- a/src/lib/publication-section-fetch.ts +++ b/src/lib/publication-section-fetch.ts @@ -1,11 +1,12 @@ import logger from '@/lib/logger' -import { FAST_READ_RELAY_URLS } from '@/constants' +import { DOCUMENT_RELAY_URLS, ExtendedKind, FAST_READ_RELAY_URLS } from '@/constants' import { publicationCoordinateLookupKeys, splitPublicationCoordinate } from '@/lib/publication-coordinate' import { buildComprehensiveRelayList } from '@/lib/relay-list-builder' import { normalizeUrl } from '@/lib/url' import client, { queryService } from '@/services/client.service' +import indexedDb from '@/services/indexed-db.service' import type { Event, Filter } from 'nostr-tools' -import { nip19 } from 'nostr-tools' +import { kinds, nip19 } from 'nostr-tools' export type PublicationSectionRef = { type: 'a' | 'e' @@ -65,6 +66,47 @@ function collectRelayHints(refs: PublicationSectionRef[]): string[] { return [...new Set(out)] } +const PUBLICATION_SECTION_QUERY_OPTS = { + globalTimeout: 22_000, + eoseTimeout: 5_000, + /** Document relays (thecitadel, etc.) are often slower than fast-read mirrors. */ + firstRelayResultGraceMs: 4_000, + foreground: true +} as const + +const PUBLICATION_CONTENT_KINDS = [ + ExtendedKind.PUBLICATION_CONTENT, + ExtendedKind.PUBLICATION, + ExtendedKind.WIKI_ARTICLE, + kinds.LongFormArticle +] as const + +async function seedSectionsFromLocalCache( + refs: PublicationSectionRef[] +): Promise> { + const out = new Map() + for (const ref of refs) { + const key = publicationRefKey(ref) + if (!key || out.has(key)) continue + try { + if (ref.type === 'a' && ref.coordinate) { + const ev = await indexedDb.getPublicationEvent(ref.coordinate) + if (ev) out.set(key, ev) + } else if (ref.type === 'e' && ref.eventId) { + const hex = resolvePublicationEventIdToHex(ref.eventId) + if (!hex) continue + const ev = + (await indexedDb.getEventFromPublicationStore(hex)) ?? + (await client.fetchEvent(hex).catch(() => undefined)) + if (ev) out.set(key, ev) + } + } catch { + // ignore per-ref cache misses + } + } + return out +} + export async function buildPublicationSectionRelayUrls( indexEvent: Event, refs: PublicationSectionRef[], @@ -72,6 +114,7 @@ export async function buildPublicationSectionRelayUrls( includeSearchableRelays = false ): Promise { const hints = collectRelayHints(refs) + const documentRelays = DOCUMENT_RELAY_URLS.map((u) => normalizeUrl(u) || u).filter((u) => !!u) const fastReadRelays = FAST_READ_RELAY_URLS.map((u) => normalizeUrl(u) || u).filter((u) => !!u) const seenOnRelays = queryService .getSeenEventRelayUrls(indexEvent.id) @@ -80,7 +123,7 @@ export async function buildPublicationSectionRelayUrls( const urls = await buildComprehensiveRelayList({ authorPubkey: indexEvent.pubkey, userPubkey: client.pubkey || undefined, - relayHints: [...hints, ...seenOnRelays], + relayHints: [...documentRelays, ...hints, ...seenOnRelays], includeUserOwnRelays: true, includeProfileFetchRelays: true, includeFastReadRelays: true, @@ -88,8 +131,10 @@ export async function buildPublicationSectionRelayUrls( includeFavoriteRelays: true, includeLocalRelays: true }) - // Keep fast-read relays pinned at the front so slicing can never drop them. - const prioritized = [...new Set([...fastReadRelays, ...hints, ...seenOnRelays, ...urls])] + // Pin document relays first — 30040/30041 content lives there, not on fast-read mirrors. + const prioritized = [ + ...new Set([...documentRelays, ...hints, ...seenOnRelays, ...fastReadRelays, ...urls]) + ] if (import.meta.env.DEV) { logger.info('[PublicationSection] relay_urls_built', { indexId: indexEvent.id, @@ -129,14 +174,19 @@ export async function batchFetchPublicationSectionEvents( refs: PublicationSectionRef[], relayUrls: string[] ): Promise> { - const out = new Map() - if (refs.length === 0 || relayUrls.length === 0) return out + const out = await seedSectionsFromLocalCache(refs) + if (refs.length === 0) return out + + const unresolvedForNetwork = refs.filter((r) => !out.has(publicationRefKey(r))) + if (unresolvedForNetwork.length === 0 || relayUrls.length === 0) return out const eRefs: PublicationSectionRef[] = [] const eHexByKey = new Map() - const aRefs = refs.filter((r) => r.type === 'a' && r.coordinate && r.pubkey && typeof r.kind === 'number') + const aRefs = unresolvedForNetwork.filter( + (r) => r.type === 'a' && r.coordinate && r.pubkey && typeof r.kind === 'number' + ) - for (const ref of refs) { + for (const ref of unresolvedForNetwork) { // Only explicit `e` refs are resolved by id. For `a` refs, tag[3] is historization metadata only. if (ref.type !== 'e' || !ref.eventId) continue const key = publicationRefKey(ref) @@ -199,11 +249,7 @@ export async function batchFetchPublicationSectionEvents( let events: Event[] = [] if (filters.length > 0) { try { - events = await queryService.fetchEvents(relayUrls, filters, { - globalTimeout: 12_000, - eoseTimeout: 2_000, - firstRelayResultGraceMs: false - }) + events = await queryService.fetchEvents(relayUrls, filters, PUBLICATION_SECTION_QUERY_OPTS) } catch (err) { if (import.meta.env.DEV) { logger.warn('[PublicationSection] batch_fetch_error', { @@ -437,12 +483,22 @@ export async function batchFetchPublicationSectionEvents( } g.dTags.push(d) } + const kindsForFallback = [ + ...new Set( + unresolvedAfterHint + .map((r) => r.kind) + .filter((k): k is number => typeof k === 'number' && k > 0) + ) + ] + const fallbackKinds = + kindsForFallback.length > 0 ? kindsForFallback : [...PUBLICATION_CONTENT_KINDS] for (const g of groups.values()) { const uniqueD = [...new Set(g.dTags)] for (let i = 0; i < uniqueD.length; i += D_CHUNK) { const dChunk = uniqueD.slice(i, i + D_CHUNK) fallbackFilters.push({ authors: [g.pubkey], + kinds: fallbackKinds, '#d': dChunk, limit: dChunk.length * ANY_KIND_LIMIT_PER_D }) @@ -457,11 +513,11 @@ export async function batchFetchPublicationSectionEvents( }) } try { - const fallbackEvents = await queryService.fetchEvents(relayUrls, fallbackFilters, { - globalTimeout: 10_000, - eoseTimeout: 2_000, - firstRelayResultGraceMs: false - }) + const fallbackEvents = await queryService.fetchEvents( + relayUrls, + fallbackFilters, + PUBLICATION_SECTION_QUERY_OPTS + ) const byAuthorD = new Map() for (const ev of fallbackEvents) { const d = dTagOf(ev) @@ -533,11 +589,11 @@ export async function batchFetchPublicationSectionEvents( }) } try { - const scanEvents = await queryService.fetchEvents(relayUrls, scanFilters, { - globalTimeout: 12_000, - eoseTimeout: 2_000, - firstRelayResultGraceMs: false - }) + const scanEvents = await queryService.fetchEvents( + relayUrls, + scanFilters, + PUBLICATION_SECTION_QUERY_OPTS + ) const scanByCoord = new Map() for (const ev of scanEvents) { const coord = coordinateOfEvent(ev) diff --git a/src/lib/publication-section-tree.test.ts b/src/lib/publication-section-tree.test.ts index 6ef1ecc4..b388aa80 100644 --- a/src/lib/publication-section-tree.test.ts +++ b/src/lib/publication-section-tree.test.ts @@ -117,6 +117,23 @@ describe('buildPublicationSectionTree', () => { expect(contentOrder).toEqual(toc.map((e) => e.title)) }) + it('uppercases Roman numerals in section titles', () => { + const c1 = `30041:${PK}:chapter-iii` + const root = indexEvent( + [ + ['d', 'book'], + ['title', 'Book'], + ['a', c1] + ], + 'root-id' + ) + const ch1 = sectionEvent('chapter-iii', 'Chapitre Iii', 'ch1-id') + const fetched = new Map([[c1, ch1]]) + + const tree = buildPublicationSectionTree(root, fetched) + expect(tree[0]?.title).toBe('Chapitre III') + }) + it('orderedPublicationRefsFromIndex assigns tagOrder in tag-list sequence', () => { const root = indexEvent( [ diff --git a/src/lib/publication-section-tree.ts b/src/lib/publication-section-tree.ts index 17cb6403..92dc2e95 100644 --- a/src/lib/publication-section-tree.ts +++ b/src/lib/publication-section-tree.ts @@ -1,5 +1,6 @@ import { ExtendedKind } from '@/constants' import { eventTagAddress } from '@/lib/publication-index' +import { uppercaseRomanNumeralsInText } from '@/lib/roman-numeral-display' import { orderedPublicationRefsFromIndex } from '@/lib/publication-asciidoc-assembler' import { publicationRefKey, @@ -65,6 +66,10 @@ function humanizeIdentifier(identifier: string): string | undefined { return identifier.replace(/-/g, ' ') } +function finalizeSectionTitle(title: string): string { + return uppercaseRomanNumeralsInText(title) +} + export function sectionTitle( ref: PublicationSectionRef, event: Event | undefined, @@ -72,23 +77,23 @@ export function sectionTitle( ): string { if (event) { const title = tagValue(event, 'title') - if (title) return title + if (title) return finalizeSectionTitle(title) const dTag = tagValue(event, 'd') const humanizedD = dTag ? humanizeIdentifier(dTag) : undefined - if (humanizedD) return humanizedD + if (humanizedD) return finalizeSectionTitle(humanizedD) } const coordinate = coordinateForRef(ref, event) if (coordinate) { const label = labelMap.get(coordinate) - if (label) return label + if (label) return finalizeSectionTitle(label) } const identifier = ref.identifier ?? (coordinate ? coordinate.split(':').slice(2).join(':') : undefined) const humanized = identifier ? humanizeIdentifier(identifier) : undefined - if (humanized) return humanized + if (humanized) return finalizeSectionTitle(humanized) return 'Section' } diff --git a/src/lib/roman-numeral-display.test.ts b/src/lib/roman-numeral-display.test.ts new file mode 100644 index 00000000..d9dd0f62 --- /dev/null +++ b/src/lib/roman-numeral-display.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { isRomanNumeralToken, uppercaseRomanNumeralsInText } from '@/lib/roman-numeral-display' + +describe('roman-numeral-display', () => { + it('detects common chapter numerals', () => { + expect(isRomanNumeralToken('iii')).toBe(true) + expect(isRomanNumeralToken('Iv')).toBe(true) + expect(isRomanNumeralToken('XII')).toBe(true) + expect(isRomanNumeralToken('Introduction')).toBe(false) + }) + + it('uppercases title-cased Roman numerals in section titles', () => { + expect(uppercaseRomanNumeralsInText('Chapitre Ii')).toBe('Chapitre II') + expect(uppercaseRomanNumeralsInText('Chapitre Iii')).toBe('Chapitre III') + expect(uppercaseRomanNumeralsInText('Chapitre Iv')).toBe('Chapitre IV') + expect(uppercaseRomanNumeralsInText('Chapitre Vi')).toBe('Chapitre VI') + expect(uppercaseRomanNumeralsInText('Chapitre Vii')).toBe('Chapitre VII') + expect(uppercaseRomanNumeralsInText('Chapitre Viii')).toBe('Chapitre VIII') + expect(uppercaseRomanNumeralsInText('Chapitre Ix')).toBe('Chapitre IX') + expect(uppercaseRomanNumeralsInText('Chapter XII')).toBe('Chapter XII') + expect(uppercaseRomanNumeralsInText('Introduction')).toBe('Introduction') + }) +}) diff --git a/src/lib/roman-numeral-display.ts b/src/lib/roman-numeral-display.ts new file mode 100644 index 00000000..3fbd4c26 --- /dev/null +++ b/src/lib/roman-numeral-display.ts @@ -0,0 +1,19 @@ +/** Token is letters I/V/X/L/C/D/M only (case-insensitive). */ +const ROMAN_LETTERS_ONLY = /^[mdclxvi]+$/i + +/** Standard 1–3999 Roman numeral grammar. */ +const ROMAN_NUMERAL_GRAMMAR = + /^(?=[mdclxvi])m{0,4}(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$/i + +export function isRomanNumeralToken(word: string): boolean { + if (!word || !ROMAN_LETTERS_ONLY.test(word)) return false + return ROMAN_NUMERAL_GRAMMAR.test(word) +} + +/** Uppercase Roman-numeral words in titles (e.g. "Chapitre Iii" → "Chapitre III"). */ +export function uppercaseRomanNumeralsInText(text: string): string { + return text.replace(/\b([mdclxvi]{1,8})\b/gi, (match, word: string) => { + if (!isRomanNumeralToken(word)) return match + return word.toUpperCase() + }) +} diff --git a/src/pages/primary/LibraryPage/index.tsx b/src/pages/primary/LibraryPage/index.tsx index 0b964575..4aa3585d 100644 --- a/src/pages/primary/LibraryPage/index.tsx +++ b/src/pages/primary/LibraryPage/index.tsx @@ -97,7 +97,10 @@ const LibraryPage = forwardRef((_props, ref) => { ) : null}