From 343230e8e65e65eb2270ec2f3a66d1795e6e6755 Mon Sep 17 00:00:00 2001 From: Silberengel Date: Sun, 7 Jun 2026 16:35:27 +0200 Subject: [PATCH] bug-fix --- src/hooks/useLibraryPublications.ts | 39 ++- src/i18n/locales/de.ts | 1 + src/i18n/locales/en.ts | 1 + src/lib/index-relay-http.ts | 6 +- src/lib/library-publication-index.test.ts | 8 +- src/lib/library-publication-index.ts | 381 +++++++++++++++++----- src/lib/publication-index.test.ts | 11 +- src/lib/publication-index.ts | 85 ++++- src/pages/primary/LibraryPage/index.tsx | 3 + 9 files changed, 441 insertions(+), 94 deletions(-) diff --git a/src/hooks/useLibraryPublications.ts b/src/hooks/useLibraryPublications.ts index 3e44a9eb..9663472e 100644 --- a/src/hooks/useLibraryPublications.ts +++ b/src/hooks/useLibraryPublications.ts @@ -6,10 +6,12 @@ import { loadLibraryPublicationIndex, type LibraryPublicationEntry } from '@/lib/library-publication-index' +import logger from '@/lib/logger' import { useNostr } from '@/providers/NostrProvider' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' const SEARCH_DEBOUNCE_MS = 300 +const LOAD_TIMEOUT_MS = 90_000 export function useLibraryPublications(isActive: boolean) { const { pubkey } = useNostr() @@ -22,6 +24,7 @@ export function useLibraryPublications(isActive: boolean) { const [allIndexCount, setAllIndexCount] = useState(0) const [topLevelCount, setTopLevelCount] = useState(0) const loadGenRef = useRef(0) + const inFlightRef = useRef(0) useEffect(() => { const t = window.setTimeout(() => setDebouncedSearch(searchQuery), SEARCH_DEBOUNCE_MS) @@ -31,20 +34,42 @@ export function useLibraryPublications(isActive: boolean) { const load = useCallback( async (forceRefresh = false) => { const gen = ++loadGenRef.current + inFlightRef.current += 1 setLoading(true) setError(null) + if (import.meta.env.DEV) { + logger.info('[Library] page load requested', { forceRefresh, gen }) + } try { const relays = await buildLibraryRelayUrls(pubkey || undefined) - const result = await loadLibraryPublicationIndex(relays, { forceRefresh }) - if (gen !== loadGenRef.current) return - setEntries(result.engaged) - setAllIndexCount(result.allIndexCount) - setTopLevelCount(result.topLevelCount) + let timeoutId: number | undefined + const timeoutPromise = new Promise((_, reject) => { + timeoutId = window.setTimeout(() => reject(new Error('Library load timed out')), LOAD_TIMEOUT_MS) + }) + try { + const result = await Promise.race([ + loadLibraryPublicationIndex(relays, { forceRefresh }), + timeoutPromise + ]) + if (gen !== loadGenRef.current) return + setEntries(result.engaged) + setAllIndexCount(result.allIndexCount) + setTopLevelCount(result.topLevelCount) + } finally { + if (timeoutId != null) window.clearTimeout(timeoutId) + } } catch (e) { if (gen !== loadGenRef.current) return - setError(e instanceof Error ? e.message : 'Failed to load library') + const message = e instanceof Error ? e.message : 'Failed to load library' + setError(message) + if (import.meta.env.DEV) { + logger.warn('[Library] page load failed', { message, gen }) + } } finally { - if (gen === loadGenRef.current) setLoading(false) + inFlightRef.current = Math.max(0, inFlightRef.current - 1) + if (inFlightRef.current === 0) { + setLoading(false) + } } }, [pubkey] diff --git a/src/i18n/locales/de.ts b/src/i18n/locales/de.ts index 3aafaf9e..b3c13f85 100644 --- a/src/i18n/locales/de.ts +++ b/src/i18n/locales/de.ts @@ -1653,6 +1653,7 @@ export default { 'Library show only my publications': 'Nur meine Publikationen', 'Library empty': 'Noch keine Publikationen mit Interaktionen auf deinen Relays.', 'Library empty filtered': 'Keine Publikationen entsprechen den Filtern.', + 'Library loading': 'Publikationen werden von Dokument-Relays geladen…', 'Library status line': '{{shown}} angezeigt · {{topLevel}} Top-Level · {{total}} Indizes geladen', 'Library badge label': 'Label', 'Library badge comment': 'Kommentar', diff --git a/src/i18n/locales/en.ts b/src/i18n/locales/en.ts index bf80dac3..f4ea8685 100644 --- a/src/i18n/locales/en.ts +++ b/src/i18n/locales/en.ts @@ -1676,6 +1676,7 @@ export default { 'Library show only my publications': 'Show only my publications', 'Library empty': 'No engaged publications found on your relays yet.', 'Library empty filtered': 'No publications match your filters.', + 'Library loading': 'Loading publications from document relays…', 'Library status line': '{{shown}} shown · {{topLevel}} top-level · {{total}} indexes loaded', 'Library badge label': 'Label', 'Library badge comment': 'Comment', diff --git a/src/lib/index-relay-http.ts b/src/lib/index-relay-http.ts index c257f463..3c431aae 100644 --- a/src/lib/index-relay-http.ts +++ b/src/lib/index-relay-http.ts @@ -188,7 +188,7 @@ function rawToVerifiedEvent(raw: Record): NEvent | null { const created_at = raw.created_at const kind = raw.kind const tags = raw.tags - const content = raw.content + const contentRaw = raw.content const sig = raw.sig if ( typeof id !== 'string' || @@ -196,11 +196,13 @@ function rawToVerifiedEvent(raw: Record): NEvent | null { typeof created_at !== 'number' || typeof kind !== 'number' || !Array.isArray(tags) || - typeof content !== 'string' || typeof sig !== 'string' ) { return null } + const content = + typeof contentRaw === 'string' ? contentRaw : contentRaw == null ? '' : null + if (content === null) return null const ev = { id, pubkey, created_at, kind, tags, content, sig } as NEvent return verifyEvent(ev) ? ev : null } catch { diff --git a/src/lib/library-publication-index.test.ts b/src/lib/library-publication-index.test.ts index f4fb8450..a71334f7 100644 --- a/src/lib/library-publication-index.test.ts +++ b/src/lib/library-publication-index.test.ts @@ -24,7 +24,7 @@ function indexEvent(d: string, aTags: string[], id = d.padEnd(64, '0').slice(0, } describe('library-publication-index', () => { - it('matches engagement on nested 30041 addresses', async () => { + it('matches engagement on nested 30041 addresses', () => { const leafAddr = `30041:${PK}:chapter-1` const childAddr = `30040:${PK}:part-1` const root = indexEvent('book', [childAddr]) @@ -42,14 +42,14 @@ describe('library-publication-index', () => { } const engagement = buildEngagementMapsFromEvents([], [], [highlight]) - const engaged = await filterEngagedPublications([root], indexByAddress, engagement, []) + const engaged = filterEngagedPublications([root], indexByAddress, engagement) expect(engaged).toHaveLength(1) expect(engaged[0].hasHighlight).toBe(true) expect(engaged[0].hasLabel).toBe(false) }) - it('matches labels by root event id', async () => { + it('matches labels by root event id', () => { const root = indexEvent('book', [`30041:${PK}:intro`]) const indexByAddress = buildIndexByAddress([root]) const label: Event = { @@ -62,7 +62,7 @@ describe('library-publication-index', () => { sig: 'e'.repeat(128) } const engagement = buildEngagementMapsFromEvents([label], [], []) - const engaged = await filterEngagedPublications([root], indexByAddress, engagement, []) + const engaged = filterEngagedPublications([root], indexByAddress, engagement) expect(engaged).toHaveLength(1) expect(engaged[0].hasLabel).toBe(true) }) diff --git a/src/lib/library-publication-index.ts b/src/lib/library-publication-index.ts index ce3d54d5..160ad510 100644 --- a/src/lib/library-publication-index.ts +++ b/src/lib/library-publication-index.ts @@ -1,20 +1,38 @@ import { ExtendedKind, LIBRARY_RELAY_URLS } from '@/constants' +import logger from '@/lib/logger' +import { queryIndexRelay } from '@/lib/index-relay-http' import { buildIndexByAddress, - collectReachableAddresses, + collectPublicationIndexEventIds, + collectReachableAddressesCached, eventTagAddress, - fetchMissingIndexByAddress, filterValidIndexEvents, - getTopLevelIndexEvents + getTopLevelIndexEvents, + hydrateNestedIndexEvents } from '@/lib/publication-index' import { buildComprehensiveRelayList } from '@/lib/relay-list-builder' -import { normalizeUrl } from '@/lib/url' +import { + canonicalRelaySessionKey, + httpIndexBasesForRelayQuery, + normalizeHttpRelayUrl, + normalizeUrl +} from '@/lib/url' import { queryService } from '@/services/client.service' import type { Event, Filter } from 'nostr-tools' import { kinds, nip19 } from 'nostr-tools' const INDEX_FETCH_LIMIT = 500 -const ENGAGEMENT_FETCH_LIMIT = 500 +const INDEX_HTTP_PAGE_LIMIT = 100 +const INDEX_HTTP_MAX_PAGES = 5 +const ENGAGEMENT_ADDRESS_CHUNK = 36 +const ENGAGEMENT_EVENT_ID_CHUNK = 44 +const MAX_TARGET_ADDRESSES = 480 +const HYDRATE_MISSING_CAP = 64 +const QUERY_OPTS = { + globalTimeout: 18_000, + eoseTimeout: 3_000, + firstRelayResultGraceMs: false as const +} export type PublicationEngagementMaps = { labelAddresses: Set @@ -44,17 +62,18 @@ function relaySetKey(urls: string[]): string { return [...new Set(urls.map((u) => normalizeUrl(u) || u))].sort().join('|') } -export async function buildLibraryRelayUrls(userPubkey?: string): Promise { - const base = LIBRARY_RELAY_URLS.map((u) => normalizeUrl(u) || u).filter(Boolean) - const urls = await buildComprehensiveRelayList({ - userPubkey, - includeUserOwnRelays: true, - includeFastReadRelays: true, - includeSearchableRelays: true, - includeFavoriteRelays: true, - relayHints: base - }) - return [...new Set([...base, ...urls])] +function splitWsAndHttpRelays(relayUrls: string[]): { wsRelays: string[]; httpRelays: string[] } { + const httpKeys = new Set( + httpIndexBasesForRelayQuery(relayUrls, []).map((u) => canonicalRelaySessionKey(u)) + ) + const wsRelays: string[] = [] + const httpRelays: string[] = [] + for (const url of relayUrls) { + const key = canonicalRelaySessionKey(normalizeUrl(url) || url) + if (httpKeys.has(key)) httpRelays.push(url) + else if (!/^https?:\/\//i.test(url.trim())) wsRelays.push(url) + } + return { wsRelays, httpRelays } } function dedupeEventsById(events: Event[]): Event[] { @@ -66,53 +85,167 @@ function dedupeEventsById(events: Event[]): Event[] { return [...byId.values()] } +function chunkArray(items: T[], size: number): T[][] { + const out: T[][] = [] + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)) + return out +} + +async function fetchPaginatedFromHttpIndexRelay(baseUrl: string, filter: Filter): Promise { + const out: Event[] = [] + const seen = new Set() + let until: number | undefined + + for (let page = 0; page < INDEX_HTTP_MAX_PAGES; page++) { + const pageFilter: Filter = { + ...filter, + limit: INDEX_HTTP_PAGE_LIMIT, + ...(until != null ? { until } : {}) + } + const batch = await queryIndexRelay(baseUrl, pageFilter) + if (batch.length === 0) break + + let oldest = batch[0].created_at + for (const ev of batch) { + if (ev.created_at < oldest) oldest = ev.created_at + if (seen.has(ev.id)) continue + seen.add(ev.id) + out.push(ev) + } + + if (batch.length < INDEX_HTTP_PAGE_LIMIT) break + until = oldest - 1 + } + + return out +} + +function normalizeLibraryRelayUrl(url: string): string { + const trimmed = url.trim() + if (!trimmed) return '' + const http = normalizeHttpRelayUrl(trimmed) + if (http) return http + return normalizeUrl(trimmed) || trimmed +} + +function libraryIndexRelayUrls(extraRelayUrls: string[] = []): string[] { + const base = LIBRARY_RELAY_URLS.map(normalizeLibraryRelayUrl).filter(Boolean) + const extra = extraRelayUrls.map(normalizeLibraryRelayUrl).filter(Boolean) + return [...new Set([...base, ...extra])] +} + +export async function buildLibraryRelayUrls(userPubkey?: string): Promise { + const base = libraryIndexRelayUrls() + const urls = await buildComprehensiveRelayList({ + userPubkey, + includeUserOwnRelays: true, + includeFastReadRelays: false, + includeSearchableRelays: false, + includeFavoriteRelays: false, + relayHints: base + }) + return libraryIndexRelayUrls([...urls]) +} + export async function fetchLibraryIndexEvents(relayUrls: string[]): Promise { - if (relayUrls.length === 0) return [] + const indexRelays = libraryIndexRelayUrls(relayUrls) + if (indexRelays.length === 0) return [] const filter: Filter = { kinds: [ExtendedKind.PUBLICATION], limit: INDEX_FETCH_LIMIT } - const events = await queryService.fetchEvents(relayUrls, [filter], { - globalTimeout: 25_000, - eoseTimeout: 4_000, - firstRelayResultGraceMs: false - }) - return filterValidIndexEvents(dedupeEventsById(events)) + const { wsRelays, httpRelays } = splitWsAndHttpRelays(indexRelays) + + const batches: Promise[] = [] + if (wsRelays.length > 0) { + batches.push(queryService.fetchEvents(wsRelays, [filter], QUERY_OPTS)) + } + for (const httpRelay of httpRelays) { + batches.push(fetchPaginatedFromHttpIndexRelay(httpRelay, filter)) + } + + const merged = dedupeEventsById((await Promise.all(batches)).flat()) + const valid = filterValidIndexEvents(merged) + if (import.meta.env.DEV) { + logger.info('[Library] index fetch', { + indexRelays: indexRelays.length, + wsRelays: wsRelays.length, + httpRelays: httpRelays.length, + mergedCount: merged.length, + validCount: valid.length + }) + } + return valid } export function buildEngagementMapsFromEvents( labels: Event[], comments: Event[], - highlights: Event[] + highlights: Event[], + targetAddresses?: Set, + targetEventIds?: Set ): PublicationEngagementMaps { const labelAddresses = new Set() const labelEventIds = new Set() const commentAddresses = new Set() const highlightAddresses = new Set() + const addressMatches = (addr: string) => !targetAddresses || targetAddresses.has(addr) + const eventIdMatches = (id: string) => !targetEventIds || targetEventIds.has(id.toLowerCase()) + for (const ev of labels) { for (const tag of ev.tags) { - if (tag[0] === 'a' && tag[1]) labelAddresses.add(tag[1]) - if (tag[0] === 'e' && tag[1]) labelEventIds.add(tag[1].toLowerCase()) + if (tag[0] === 'a' && tag[1] && addressMatches(tag[1])) labelAddresses.add(tag[1]) + if (tag[0] === 'e' && tag[1] && eventIdMatches(tag[1])) labelEventIds.add(tag[1].toLowerCase()) } } for (const ev of comments) { for (const tag of ev.tags) { - if (tag[0] === 'A' && tag[1]) commentAddresses.add(tag[1]) + if (tag[0] === 'A' && tag[1] && addressMatches(tag[1])) commentAddresses.add(tag[1]) } } for (const ev of highlights) { for (const tag of ev.tags) { - if (tag[0] === 'a' && tag[1]) highlightAddresses.add(tag[1]) + if (tag[0] === 'a' && tag[1] && addressMatches(tag[1])) highlightAddresses.add(tag[1]) } } return { labelAddresses, labelEventIds, commentAddresses, highlightAddresses } } +async function fetchHttpEngagementByAddresses( + httpRelays: string[], + kind: number, + tagKey: '#a' | '#A', + addressChunks: string[][] +): Promise { + if (httpRelays.length === 0 || addressChunks.length === 0) return [] + const out: Event[] = [] + const seen = new Set() + for (const relay of httpRelays) { + for (const chunk of addressChunks) { + if (chunk.length === 0) continue + const filter = { + kinds: [kind], + [tagKey]: chunk, + limit: Math.min(chunk.length * 10, INDEX_HTTP_PAGE_LIMIT) + } as Filter + const batch = await queryIndexRelay(relay, filter) + for (const ev of batch) { + if (seen.has(ev.id)) continue + seen.add(ev.id) + out.push(ev) + } + } + } + return out +} + export async function fetchPublicationEngagementMaps( - relayUrls: string[] + relayUrls: string[], + targetAddresses: Set, + targetEventIds: Set ): Promise { - if (relayUrls.length === 0) { + if (relayUrls.length === 0 || targetAddresses.size === 0) { return { labelAddresses: new Set(), labelEventIds: new Set(), @@ -121,34 +254,59 @@ export async function fetchPublicationEngagementMaps( } } - const opts = { - globalTimeout: 25_000, - eoseTimeout: 4_000, - firstRelayResultGraceMs: false as const - } + const addressChunks = chunkArray([...targetAddresses], ENGAGEMENT_ADDRESS_CHUNK) + const eventIdChunks = chunkArray([...targetEventIds], ENGAGEMENT_EVENT_ID_CHUNK) + const { wsRelays, httpRelays } = splitWsAndHttpRelays(relayUrls) - const [labels, comments, highlights] = await Promise.all([ - queryService.fetchEvents( - relayUrls, - [{ kinds: [ExtendedKind.LABEL], limit: ENGAGEMENT_FETCH_LIMIT }], - opts - ), - queryService.fetchEvents( - relayUrls, - [{ kinds: [ExtendedKind.COMMENT], limit: ENGAGEMENT_FETCH_LIMIT }], - opts - ), - queryService.fetchEvents( - relayUrls, - [{ kinds: [kinds.Highlights], limit: ENGAGEMENT_FETCH_LIMIT }], - opts - ) + const highlightFilters = addressChunks.map( + (chunk): Filter => ({ kinds: [kinds.Highlights], '#a': chunk, limit: chunk.length * 12 }) + ) + const labelAddressFilters = addressChunks.map( + (chunk): Filter => ({ kinds: [ExtendedKind.LABEL], '#a': chunk, limit: chunk.length * 8 }) + ) + const labelEventFilters = eventIdChunks.map( + (chunk): Filter => ({ kinds: [ExtendedKind.LABEL], '#e': chunk, limit: chunk.length * 6 }) + ) + const commentWsFilters = addressChunks.map( + (chunk): Filter => ({ kinds: [ExtendedKind.COMMENT], '#A': chunk, limit: chunk.length * 12 }) + ) + + const highlightPromise = Promise.all([ + wsRelays.length > 0 && highlightFilters.length > 0 + ? queryService.fetchEvents(wsRelays, highlightFilters, QUERY_OPTS) + : Promise.resolve([] as Event[]), + fetchHttpEngagementByAddresses(httpRelays, kinds.Highlights, '#a', addressChunks) + ]).then(([scoped, bulk]) => dedupeEventsById([...scoped, ...bulk])) + + const labelPromise = Promise.all([ + wsRelays.length > 0 && labelAddressFilters.length > 0 + ? queryService.fetchEvents(wsRelays, labelAddressFilters, QUERY_OPTS) + : Promise.resolve([] as Event[]), + wsRelays.length > 0 && labelEventFilters.length > 0 + ? queryService.fetchEvents(wsRelays, labelEventFilters, QUERY_OPTS) + : Promise.resolve([] as Event[]), + fetchHttpEngagementByAddresses(httpRelays, ExtendedKind.LABEL, '#a', addressChunks) + ]).then(([byAddress, byEvent, bulk]) => dedupeEventsById([...byAddress, ...byEvent, ...bulk])) + + const commentPromise = Promise.all([ + wsRelays.length > 0 && commentWsFilters.length > 0 + ? queryService.fetchEvents(wsRelays, commentWsFilters, QUERY_OPTS) + : Promise.resolve([] as Event[]), + fetchHttpEngagementByAddresses(httpRelays, ExtendedKind.COMMENT, '#A', addressChunks) + ]).then(([scoped, bulk]) => dedupeEventsById([...scoped, ...bulk])) + + const [highlights, labels, comments] = await Promise.all([ + highlightPromise, + labelPromise, + commentPromise ]) return buildEngagementMapsFromEvents( dedupeEventsById(labels), dedupeEventsById(comments), - dedupeEventsById(highlights) + dedupeEventsById(highlights), + targetAddresses, + targetEventIds ) } @@ -165,17 +323,15 @@ function addressHasEngagement( return { hasLabel, hasComment, hasHighlight } } -export async function filterEngagedPublications( +export function filterEngagedPublications( roots: Event[], indexByAddress: Map, - engagement: PublicationEngagementMaps, - relayUrls: string[] -): Promise { - const fetchMissing = (address: string) => fetchMissingIndexByAddress(address, relayUrls) + engagement: PublicationEngagementMaps +): LibraryPublicationEntry[] { const out: LibraryPublicationEntry[] = [] for (const root of roots) { - const reachable = await collectReachableAddresses(root, indexByAddress, fetchMissing) + const reachable = collectReachableAddressesCached(root, indexByAddress) const rootAddr = eventTagAddress(root) if (rootAddr) reachable.add(rootAddr) @@ -278,6 +434,41 @@ export function filterLibraryPublicationsByUser( }) } +function collectTargetAddressesFromIndexes( + indexEvents: Event[], + indexByAddress: Map +): Set { + const addresses = new Set() + outer: for (const root of getTopLevelIndexEvents(indexEvents)) { + for (const addr of collectReachableAddressesCached(root, indexByAddress)) { + addresses.add(addr) + if (addresses.size >= MAX_TARGET_ADDRESSES) break outer + } + const rootAddr = eventTagAddress(root) + if (rootAddr) { + addresses.add(rootAddr) + if (addresses.size >= MAX_TARGET_ADDRESSES) break outer + } + } + return addresses +} + +async function buildEngagedFromCache( + relayUrls: string[], + indexEvents: Event[], + indexByAddress: Map, + engagement?: PublicationEngagementMaps +): Promise { + const topLevel = getTopLevelIndexEvents(indexEvents) + let maps = engagement + if (!maps) { + const targetAddresses = collectTargetAddressesFromIndexes(indexEvents, indexByAddress) + const targetEventIds = collectPublicationIndexEventIds(indexEvents) + maps = await fetchPublicationEngagementMaps(relayUrls, targetAddresses, targetEventIds) + } + return sortLibraryPublications(filterEngagedPublications(topLevel, indexByAddress, maps)) +} + export async function loadLibraryPublicationIndex( relayUrls: string[], options?: { forceRefresh?: boolean } @@ -287,32 +478,76 @@ export async function loadLibraryPublicationIndex( topLevelCount: number }> { const key = relaySetKey(relayUrls) + if (import.meta.env.DEV) { + logger.info('[Library] load start', { relayCount: relayUrls.length, cached: sessionCache?.relayKey === key }) + } + if (!options?.forceRefresh && sessionCache?.relayKey === key) { + const engaged = await buildEngagedFromCache( + relayUrls, + sessionCache.indexEvents, + sessionCache.indexByAddress, + sessionCache.engagement + ) + if (import.meta.env.DEV) { + logger.info('[Library] load from cache', { engaged: engaged.length }) + } return { - engaged: sortLibraryPublications( - await filterEngagedPublications( - getTopLevelIndexEvents(sessionCache.indexEvents), - sessionCache.indexByAddress, - sessionCache.engagement, - relayUrls - ) - ), + engaged, allIndexCount: sessionCache.indexEvents.length, topLevelCount: getTopLevelIndexEvents(sessionCache.indexEvents).length } } - const [indexEvents, engagement] = await Promise.all([ - fetchLibraryIndexEvents(relayUrls), - fetchPublicationEngagementMaps(relayUrls) - ]) + const indexEvents = await fetchLibraryIndexEvents(relayUrls) + if (import.meta.env.DEV) { + logger.info('[Library] indexes fetched', { validCount: indexEvents.length }) + } + const indexByAddress = buildIndexByAddress(indexEvents) + const topLevelForHydrate = getTopLevelIndexEvents(indexEvents) + await hydrateNestedIndexEvents(indexEvents, indexByAddress, relayUrls, { + maxPasses: 1, + maxMissingPerPass: HYDRATE_MISSING_CAP, + scanRoots: topLevelForHydrate + }) + if (import.meta.env.DEV) { + logger.info('[Library] nested hydrate done', { indexCount: indexEvents.length }) + } + + const targetAddresses = collectTargetAddressesFromIndexes(indexEvents, indexByAddress) + const targetEventIds = collectPublicationIndexEventIds(indexEvents) + if (import.meta.env.DEV) { + logger.info('[Library] fetching engagement', { + targetAddresses: targetAddresses.size, + targetEventIds: targetEventIds.size + }) + } + const engagement = await fetchPublicationEngagementMaps( + relayUrls, + targetAddresses, + targetEventIds + ) + if (import.meta.env.DEV) { + logger.info('[Library] engagement maps built', { + labels: engagement.labelAddresses.size + engagement.labelEventIds.size, + comments: engagement.commentAddresses.size, + highlights: engagement.highlightAddresses.size + }) + } + sessionCache = { relayKey: key, indexEvents, indexByAddress, engagement } const topLevel = getTopLevelIndexEvents(indexEvents) - const engaged = sortLibraryPublications( - await filterEngagedPublications(topLevel, indexByAddress, engagement, relayUrls) - ) + const engaged = sortLibraryPublications(filterEngagedPublications(topLevel, indexByAddress, engagement)) + + if (import.meta.env.DEV) { + logger.info('[Library] load done', { + engaged: engaged.length, + topLevel: topLevel.length, + allIndexCount: indexEvents.length + }) + } return { engaged, diff --git a/src/lib/publication-index.test.ts b/src/lib/publication-index.test.ts index a3fb8da6..54422aa2 100644 --- a/src/lib/publication-index.test.ts +++ b/src/lib/publication-index.test.ts @@ -3,6 +3,7 @@ import { ExtendedKind } from '@/constants' import { buildIndexByAddress, collectReachableAddresses, + collectReachableAddressesCached, eventTagAddress, filterValidIndexEvents, getTopLevelIndexEvents @@ -40,7 +41,9 @@ describe('publication-index', () => { const valid = indexEvent('book', [`30041:${PK}:chapter-1`]) const withContent = { ...valid, content: 'not empty' } const noTitle = { ...valid, tags: [['d', 'book'], ['a', `30041:${PK}:chapter-1`]] } + const nullContent = { ...valid, content: null as unknown as string } expect(filterValidIndexEvents([valid])).toHaveLength(1) + expect(filterValidIndexEvents([nullContent])).toHaveLength(1) expect(filterValidIndexEvents([withContent, noTitle])).toHaveLength(0) }) @@ -53,18 +56,14 @@ describe('publication-index', () => { expect(eventTagAddress(top[0])).toBe(`30040:${PK}:book`) }) - it('collectReachableAddresses walks nested 30040 and 30041 refs', async () => { + it('collectReachableAddressesCached walks nested 30040 and 30041 refs', () => { const childAddr = `30040:${PK}:part-1` const leafAddr = `30041:${PK}:chapter-1` const root = indexEvent('book', [childAddr, `30041:${PK}:intro`]) const child = indexEvent('part-1', [leafAddr], '2'.repeat(64)) const indexByAddress = buildIndexByAddress([root, child]) - const reachable = await collectReachableAddresses( - root, - indexByAddress, - async () => null - ) + const reachable = collectReachableAddressesCached(root, indexByAddress) expect(reachable.has(`30040:${PK}:book`)).toBe(true) expect(reachable.has(childAddr)).toBe(true) diff --git a/src/lib/publication-index.ts b/src/lib/publication-index.ts index 96674e54..cd7deed1 100644 --- a/src/lib/publication-index.ts +++ b/src/lib/publication-index.ts @@ -17,8 +17,10 @@ export function filterValidIndexEvents(events: Event[]): Event[] { return events.filter((event) => { if (event.kind !== ExtendedKind.PUBLICATION) return false if (event.content != null && event.content.length > 0) return false - const hasTitle = event.tags.some((t) => t[0] === 'title' && t[1]) - const hasD = event.tags.some((t) => t[0] === 'd' && t[1]) + const hasTitle = event.tags.some( + (t) => (t[0] || '').trim().toLowerCase() === 'title' && t[1] + ) + const hasD = event.tags.some((t) => (t[0] || '').trim().toLowerCase() === 'd' && t[1]) const hasA = event.tags.some((t) => t[0] === 'a' && t[1]) const hasE = event.tags.some((t) => t[0] === 'e' && t[1]) return hasTitle && hasD && (hasA || hasE) @@ -91,6 +93,85 @@ export function buildIndexByAddress(events: Event[]): Map { return map } +/** BFS over addresses already present in `indexByAddress` (no network I/O). */ +export function collectReachableAddressesCached( + root: Event, + indexByAddress: Map +): Set { + const reachable = new Set() + const rootAddr = eventTagAddress(root) + if (!rootAddr) return reachable + + const queue = [rootAddr] + while (queue.length > 0) { + const addr = queue.shift()! + if (reachable.has(addr)) continue + reachable.add(addr) + + const event = indexByAddress.get(addr) + if (!event || event.kind !== ExtendedKind.PUBLICATION) continue + + for (const child of collectChildAddressesFromIndex(event)) { + if (!reachable.has(child)) queue.push(child) + } + } + + return reachable +} + +export function collectPublicationIndexEventIds(events: Event[]): Set { + return new Set(events.map((ev) => ev.id.toLowerCase())) +} + +export type HydrateNestedIndexOptions = { + maxPasses?: number + /** Cap missing nested 30040 fetches per pass (library bulk load). */ + maxMissingPerPass?: number + /** When set, only scan these roots for missing nested 30040 refs. */ + scanRoots?: Event[] +} + +/** Batch-fetch nested kind 30040 indexes referenced by `a` tags but missing from cache. */ +export async function hydrateNestedIndexEvents( + indexEvents: Event[], + indexByAddress: Map, + relayUrls: string[], + options?: HydrateNestedIndexOptions | number +): Promise { + const opts: HydrateNestedIndexOptions = + typeof options === 'number' ? { maxPasses: options } : (options ?? {}) + const maxPasses = opts.maxPasses ?? 2 + const maxMissingPerPass = opts.maxMissingPerPass + const scanEvents = opts.scanRoots ?? indexEvents + + for (let pass = 0; pass < maxPasses; pass++) { + const missingRefs: PublicationSectionRef[] = [] + const seenCoords = new Set() + for (const event of scanEvents) { + if (event.kind !== ExtendedKind.PUBLICATION) continue + for (const ref of collectPublicationATagRefs(event)) { + if (ref.kind !== ExtendedKind.PUBLICATION || !ref.coordinate) continue + if (indexByAddress.has(ref.coordinate) || seenCoords.has(ref.coordinate)) continue + seenCoords.add(ref.coordinate) + missingRefs.push(ref) + if (maxMissingPerPass != null && missingRefs.length >= maxMissingPerPass) break + } + if (maxMissingPerPass != null && missingRefs.length >= maxMissingPerPass) break + } + if (missingRefs.length === 0) break + const fetched = await batchFetchPublicationSectionEvents(missingRefs, relayUrls) + let added = 0 + for (const ev of fetched.values()) { + const addr = eventTagAddress(ev) + if (!addr || indexByAddress.has(addr)) continue + indexByAddress.set(addr, ev) + indexEvents.push(ev) + added++ + } + if (added === 0) break + } +} + export async function collectReachableAddresses( root: Event, indexByAddress: Map, diff --git a/src/pages/primary/LibraryPage/index.tsx b/src/pages/primary/LibraryPage/index.tsx index a861377f..58a13d7b 100644 --- a/src/pages/primary/LibraryPage/index.tsx +++ b/src/pages/primary/LibraryPage/index.tsx @@ -67,6 +67,9 @@ const LibraryPage = forwardRef((_props, ref) => { {error} ) : null} + {loading ? ( +

{t('Library loading')}

+ ) : null} {statusLine ? (

{statusLine}

) : null}