diff --git a/src/hooks/useLibraryPublications.ts b/src/hooks/useLibraryPublications.ts index 2cfd782b..c932e5c7 100644 --- a/src/hooks/useLibraryPublications.ts +++ b/src/hooks/useLibraryPublications.ts @@ -11,7 +11,7 @@ import { useNostr } from '@/providers/NostrProvider' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' const SEARCH_DEBOUNCE_MS = 300 -const LOAD_TIMEOUT_MS = 90_000 +const LOAD_TIMEOUT_MS = 120_000 export function useLibraryPublications(isActive: boolean) { const { pubkey } = useNostr() @@ -20,11 +20,11 @@ export function useLibraryPublications(isActive: boolean) { const [debouncedSearch, setDebouncedSearch] = useState('') const [showOnlyMine, setShowOnlyMine] = useState(false) const [loading, setLoading] = useState(false) + const [engagementLoading, setEngagementLoading] = useState(false) const [error, setError] = useState(null) const [allIndexCount, setAllIndexCount] = useState(0) const [topLevelCount, setTopLevelCount] = useState(0) const loadGenRef = useRef(0) - const inFlightRef = useRef(0) useEffect(() => { const t = window.setTimeout(() => setDebouncedSearch(searchQuery), SEARCH_DEBOUNCE_MS) @@ -34,8 +34,8 @@ export function useLibraryPublications(isActive: boolean) { const load = useCallback( async (forceRefresh = false) => { const gen = ++loadGenRef.current - inFlightRef.current += 1 setLoading(true) + setEngagementLoading(false) setError(null) if (import.meta.env.DEV) { logger.info('[Library] page load requested', { forceRefresh, gen }) @@ -48,7 +48,17 @@ export function useLibraryPublications(isActive: boolean) { }) try { const result = await Promise.race([ - loadLibraryPublicationIndex(relays, { forceRefresh }), + loadLibraryPublicationIndex(relays, { + forceRefresh, + onIndexesReady: (snapshot) => { + if (gen !== loadGenRef.current) return + setEntries(snapshot.engaged) + setAllIndexCount(snapshot.allIndexCount) + setTopLevelCount(snapshot.topLevelCount) + setLoading(false) + setEngagementLoading(true) + } + }), timeoutPromise ]) if (gen !== loadGenRef.current) return @@ -66,9 +76,9 @@ export function useLibraryPublications(isActive: boolean) { logger.warn('[Library] page load failed', { message, gen }) } } finally { - inFlightRef.current = Math.max(0, inFlightRef.current - 1) - if (inFlightRef.current === 0) { + if (gen === loadGenRef.current) { setLoading(false) + setEngagementLoading(false) } } }, @@ -102,6 +112,7 @@ export function useLibraryPublications(isActive: boolean) { showOnlyMine, setShowOnlyMine, loading, + engagementLoading, error, allIndexCount, topLevelCount, diff --git a/src/i18n/locales/de.ts b/src/i18n/locales/de.ts index 2d9f7a96..43f690b2 100644 --- a/src/i18n/locales/de.ts +++ b/src/i18n/locales/de.ts @@ -1654,6 +1654,7 @@ export default { 'Library empty': 'Noch keine Publikationen auf deinen Relays gefunden.', 'Library empty filtered': 'Keine Publikationen entsprechen den Filtern.', 'Library loading': 'Publikationen werden von Dokument-Relays geladen…', + 'Library engagement loading': 'Engagement-Filter werden aktualisiert…', 'Library status line': '{{shown}} angezeigt · {{topLevel}} Top-Level · {{total}} Indizes geladen', 'Library badge label': 'Label', 'Library badge comment': 'Kommentar', diff --git a/src/i18n/locales/en.ts b/src/i18n/locales/en.ts index b78e33e9..12081952 100644 --- a/src/i18n/locales/en.ts +++ b/src/i18n/locales/en.ts @@ -1677,6 +1677,7 @@ export default { 'Library empty': 'No publications found on your relays yet.', 'Library empty filtered': 'No publications match your filters.', 'Library loading': 'Loading publications from document relays…', + 'Library engagement loading': 'Updating engagement filters…', 'Library status line': '{{shown}} shown · {{topLevel}} top-level · {{total}} indexes loaded', 'Library badge label': 'Label', 'Library badge comment': 'Comment', diff --git a/src/lib/index-relay-http.test.ts b/src/lib/index-relay-http.test.ts index 66765100..4ec9a090 100644 --- a/src/lib/index-relay-http.test.ts +++ b/src/lib/index-relay-http.test.ts @@ -2,8 +2,10 @@ import { IndexRelayTransportError, clearDevIndexRelayUnavailableThisSession, isDevIndexRelayUnavailableThisSession, - isIndexRelayTransportFailure + isIndexRelayTransportFailure, + rawToIndexRelayEvent } from '@/lib/index-relay-http' +import { finalizeEvent, generateSecretKey, getPublicKey } from 'nostr-tools' import { describe, expect, it, beforeEach } from 'vitest' describe('isIndexRelayTransportFailure', () => { @@ -26,3 +28,27 @@ describe('dev index relay session skip', () => { expect(isDevIndexRelayUnavailableThisSession()).toBe(false) }) }) + +describe('rawToIndexRelayEvent', () => { + it('accepts kind 30040 with empty or null content per NKBIP-01', () => { + const sk = generateSecretKey() + const pubkey = getPublicKey(sk) + const verified = finalizeEvent( + { + kind: 30040, + created_at: 1_700_000_000, + tags: [ + ['d', 'book'], + ['title', 'Test Book'], + ['a', `30041:${pubkey}:chapter-1`] + ], + content: '' + }, + sk + ) + const mercuryRow = { ...verified, content: null } as unknown as Record + const parsed = rawToIndexRelayEvent(mercuryRow) + expect(parsed?.content).toBe('') + expect(parsed?.kind).toBe(30040) + }) +}) diff --git a/src/lib/index-relay-http.ts b/src/lib/index-relay-http.ts index 3c431aae..dfb3ec8a 100644 --- a/src/lib/index-relay-http.ts +++ b/src/lib/index-relay-http.ts @@ -7,6 +7,7 @@ * Known broken CORS HTTPS hosts (e.g. nos.lol) use `/dev-cors-index-relay` (see `vite.config.ts` + `url.ts`). * Production and other remote HTTPS relays still need CORS or your own reverse proxy. */ +import { ExtendedKind } from '@/constants' import { fetchWithTimeout } from '@/lib/fetch-with-timeout' import logger from '@/lib/logger' import { @@ -15,7 +16,7 @@ import { normalizeHttpRelayUrl } from '@/lib/url' import type { Filter, Event as NEvent } from 'nostr-tools' -import { verifyEvent } from 'nostr-tools' +import { validateEvent, verifyEvent } from 'nostr-tools' function trimSlash(base: string): string { return base.replace(/\/+$/, '') @@ -181,6 +182,14 @@ function handleFilterTransportFailure(endpoint: string, err?: unknown): void { }) } +/** NKBIP-01 kind 30040 indexes always have empty `content` (relays may JSON-encode that as `null`). */ +function normalizedIndexRelayContent(kind: number, contentRaw: unknown): string | null { + if (kind === ExtendedKind.PUBLICATION) return '' + if (typeof contentRaw === 'string') return contentRaw + if (contentRaw == null) return '' + return null +} + function rawToVerifiedEvent(raw: Record): NEvent | null { try { const id = raw.id @@ -200,8 +209,7 @@ function rawToVerifiedEvent(raw: Record): NEvent | null { ) { return null } - const content = - typeof contentRaw === 'string' ? contentRaw : contentRaw == null ? '' : null + const content = normalizedIndexRelayContent(kind, contentRaw) if (content === null) return null const ev = { id, pubkey, created_at, kind, tags, content, sig } as NEvent return verifyEvent(ev) ? ev : null @@ -210,6 +218,54 @@ function rawToVerifiedEvent(raw: Record): NEvent | null { } } +/** + * Parse HTTP index relay rows for Library discovery. Kind 30040 content is always normalized to `''`. + * When verify fails (some index mirrors store stale id/sig), accept structurally valid 30040 rows. + */ +export function rawToIndexRelayEvent(raw: Record): NEvent | null { + try { + const id = raw.id + const pubkey = raw.pubkey + const created_at = raw.created_at + const kind = raw.kind + const tags = raw.tags + const contentRaw = raw.content + const sig = raw.sig + if ( + typeof id !== 'string' || + typeof pubkey !== 'string' || + typeof created_at !== 'number' || + typeof kind !== 'number' || + !Array.isArray(tags) || + typeof sig !== 'string' + ) { + return null + } + const content = normalizedIndexRelayContent(kind, contentRaw) + if (content === null) return null + const ev = { + id: id.toLowerCase(), + pubkey: pubkey.toLowerCase(), + created_at, + kind, + tags, + content, + sig + } as NEvent + if (verifyEvent(ev)) return ev + if (kind === ExtendedKind.PUBLICATION && validateEvent(ev)) return ev + return null + } catch { + return null + } +} + +export type TIndexRelayLibraryPage = { + events: NEvent[] + /** Rows returned by the relay before client-side filtering (drives pagination). */ + apiRowCount: number +} + /** * Query one HTTP index relay. Runs one POST per filter when given an array. */ @@ -299,6 +355,68 @@ export async function queryIndexRelay( return out } +/** Library discovery: paginate using {@link rawToIndexRelayEvent} and the relay's raw row count. */ +export async function queryIndexRelayForLibrary( + baseUrl: string, + filter: Filter, + options?: { signal?: AbortSignal } +): Promise { + const base = devHttpIndexRelayBaseForFetch(baseUrl) + const endpoint = indexRelayFilterUrl(base) + if (shouldSkipDevIndexRelayFetch(endpoint)) { + return { events: [], apiRowCount: 0 } + } + + const body = nostrFilterToIndexRelayBody(filterForIndexRelay(filter)) + try { + const res = await fetchWithTimeout(endpoint, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify(body), + signal: options?.signal, + timeoutMs: 25_000 + }) + if (!res.ok) { + if (res.status >= 500) { + markDevIndexRelayUnavailableFromHttpStatus(res.status, endpoint) + throw new IndexRelayTransportError(new Error(`HTTP ${res.status}`)) + } + return { events: [], apiRowCount: 0 } + } + clearDevIndexRelayUnavailableThisSession() + const json = (await res.json()) as { data?: unknown } + const data = json.data + if (!Array.isArray(data)) return { events: [], apiRowCount: 0 } + + const events: NEvent[] = [] + const seen = new Set() + for (const item of data) { + if (!item || typeof item !== 'object') continue + const ev = rawToIndexRelayEvent(item as Record) + if (ev && !seen.has(ev.id)) { + seen.add(ev.id) + events.push(ev) + } + } + return { events, apiRowCount: data.length } + } catch (e) { + if ((e as Error).name === 'AbortError') throw e + if (e instanceof IndexRelayTransportError) throw e + if (isIndexRelayTransportFailure(e)) { + handleFilterTransportFailure(endpoint, e) + throw new IndexRelayTransportError(e) + } + warnIndexRelayHttpThrottled(endpoint, '[IndexRelayHttp] library filter request error', { + endpoint, + error: e + }) + return { events: [], apiRowCount: 0 } + } +} + function filterForIndexRelay(f: Filter): Filter { const rest = { ...f } as Filter & { search?: unknown } delete rest.search diff --git a/src/lib/library-publication-index.ts b/src/lib/library-publication-index.ts index 5c49e652..42d6e7e2 100644 --- a/src/lib/library-publication-index.ts +++ b/src/lib/library-publication-index.ts @@ -1,6 +1,6 @@ import { ExtendedKind, LIBRARY_RELAY_URLS } from '@/constants' import logger from '@/lib/logger' -import { queryIndexRelay } from '@/lib/index-relay-http' +import { queryIndexRelay, queryIndexRelayForLibrary } from '@/lib/index-relay-http' import { buildIndexByAddress, collectPublicationIndexEventIds, @@ -35,7 +35,8 @@ const ENGAGEMENT_ADDRESS_CHUNK = 36 const ENGAGEMENT_EVENT_ID_CHUNK = 44 const MAX_TARGET_ADDRESSES = 480 const HYDRATE_MISSING_CAP = 64 -export const LIBRARY_RECENT_FALLBACK_LIMIT = 10 +export const LIBRARY_RECENT_FALLBACK_LIMIT = 120 +const ENGAGEMENT_FETCH_TIMEOUT_MS = 25_000 const QUERY_OPTS = { globalTimeout: 18_000, eoseTimeout: 3_000, @@ -110,10 +111,25 @@ async function fetchPaginatedFromHttpIndexRelay(baseUrl: string, filter: Filter) limit: INDEX_HTTP_PAGE_LIMIT, ...(until != null ? { until } : {}) } - const batch = await queryIndexRelay(baseUrl, pageFilter) - if (batch.length === 0) break + let batch: Event[] = [] + let apiRowCount = 0 + try { + const pageResult = await queryIndexRelayForLibrary(baseUrl, pageFilter) + batch = pageResult.events + apiRowCount = pageResult.apiRowCount + } catch (e) { + if (import.meta.env.DEV) { + logger.warn('[Library] HTTP index page failed', { + baseUrl, + page, + message: e instanceof Error ? e.message : String(e) + }) + } + break + } + if (apiRowCount === 0) break - let oldest = batch[0].created_at + let oldest = batch[0]?.created_at ?? Number.MAX_SAFE_INTEGER for (const ev of batch) { if (ev.created_at < oldest) oldest = ev.created_at if (seen.has(ev.id)) continue @@ -121,7 +137,8 @@ async function fetchPaginatedFromHttpIndexRelay(baseUrl: string, filter: Filter) out.push(ev) } - if (batch.length < INDEX_HTTP_PAGE_LIMIT) break + if (apiRowCount < INDEX_HTTP_PAGE_LIMIT) break + if (oldest === Number.MAX_SAFE_INTEGER) break until = oldest - 1 } @@ -165,14 +182,25 @@ export async function fetchLibraryIndexEvents(relayUrls: string[]): Promise[] = [] if (wsRelays.length > 0) { - batches.push(queryService.fetchEvents(wsRelays, [filter], QUERY_OPTS)) + batches.push( + queryService.fetchEvents(wsRelays, [filter], QUERY_OPTS).catch((e) => { + if (import.meta.env.DEV) { + logger.warn('[Library] WS index fetch failed', { + message: e instanceof Error ? e.message : String(e) + }) + } + return [] as Event[] + }) + ) } for (const httpRelay of httpRelays) { batches.push(fetchPaginatedFromHttpIndexRelay(httpRelay, filter)) } - const networkMerged = - batches.length > 0 ? dedupeEventsById((await Promise.all(batches)).flat()) : [] + const settled = await Promise.allSettled(batches) + const networkMerged = dedupeEventsById( + settled.flatMap((r) => (r.status === 'fulfilled' ? r.value : [])) + ) const merged = dedupeEventsById([...cached, ...networkMerged]) const valid = filterValidIndexEvents(merged) void persistLibraryIndexCacheEvents(valid) @@ -258,7 +286,8 @@ async function fetchHttpEngagementByAddresses( export async function fetchPublicationEngagementMaps( relayUrls: string[], targetAddresses: Set, - targetEventIds: Set + targetEventIds: Set, + options?: { httpOnly?: boolean } ): Promise { if (relayUrls.length === 0 || targetAddresses.size === 0) { return { @@ -272,6 +301,7 @@ export async function fetchPublicationEngagementMaps( const addressChunks = chunkArray([...targetAddresses], ENGAGEMENT_ADDRESS_CHUNK) const eventIdChunks = chunkArray([...targetEventIds], ENGAGEMENT_EVENT_ID_CHUNK) const { wsRelays, httpRelays } = splitWsAndHttpRelays(relayUrls) + const useWs = !options?.httpOnly && wsRelays.length > 0 const highlightFilters = addressChunks.map( (chunk): Filter => ({ kinds: [kinds.Highlights], '#a': chunk, limit: chunk.length * 12 }) @@ -287,24 +317,24 @@ export async function fetchPublicationEngagementMaps( ) const highlightPromise = Promise.all([ - wsRelays.length > 0 && highlightFilters.length > 0 + useWs && highlightFilters.length > 0 ? queryService.fetchEvents(wsRelays, highlightFilters, QUERY_OPTS) : Promise.resolve([] as Event[]), fetchHttpEngagementByAddresses(httpRelays, kinds.Highlights, '#a', addressChunks) ]).then(([scoped, bulk]) => dedupeEventsById([...scoped, ...bulk])) const labelPromise = Promise.all([ - wsRelays.length > 0 && labelAddressFilters.length > 0 + useWs && labelAddressFilters.length > 0 ? queryService.fetchEvents(wsRelays, labelAddressFilters, QUERY_OPTS) : Promise.resolve([] as Event[]), - wsRelays.length > 0 && labelEventFilters.length > 0 + useWs && labelEventFilters.length > 0 ? queryService.fetchEvents(wsRelays, labelEventFilters, QUERY_OPTS) : Promise.resolve([] as Event[]), fetchHttpEngagementByAddresses(httpRelays, ExtendedKind.LABEL, '#a', addressChunks) ]).then(([byAddress, byEvent, bulk]) => dedupeEventsById([...byAddress, ...byEvent, ...bulk])) const commentPromise = Promise.all([ - wsRelays.length > 0 && commentWsFilters.length > 0 + useWs && commentWsFilters.length > 0 ? queryService.fetchEvents(wsRelays, commentWsFilters, QUERY_OPTS) : Promise.resolve([] as Event[]), fetchHttpEngagementByAddresses(httpRelays, ExtendedKind.COMMENT, '#A', addressChunks) @@ -513,7 +543,15 @@ async function buildEngagedFromCache( export async function loadLibraryPublicationIndex( relayUrls: string[], - options?: { forceRefresh?: boolean } + options?: { + forceRefresh?: boolean + /** Called as soon as kind-30040 indexes are loaded — before engagement (which can take minutes). */ + onIndexesReady?: (snapshot: { + engaged: LibraryPublicationEntry[] + allIndexCount: number + topLevelCount: number + }) => void + } ): Promise<{ engaged: LibraryPublicationEntry[] allIndexCount: number @@ -547,7 +585,15 @@ export async function loadLibraryPublicationIndex( } const indexByAddress = buildIndexByAddress(indexEvents) - const topLevelForHydrate = getTopLevelIndexEvents(indexEvents) + let topLevel = getTopLevelIndexEvents(indexEvents) + + options?.onIndexesReady?.({ + engaged: buildRecentPublicationEntries(topLevel), + allIndexCount: indexEvents.length, + topLevelCount: topLevel.length + }) + + const topLevelForHydrate = topLevel await hydrateNestedIndexEvents(indexEvents, indexByAddress, relayUrls, { maxPasses: 1, maxMissingPerPass: HYDRATE_MISSING_CAP, @@ -557,6 +603,7 @@ export async function loadLibraryPublicationIndex( logger.info('[Library] nested hydrate done', { indexCount: indexEvents.length }) } + topLevel = getTopLevelIndexEvents(indexEvents) const targetAddresses = collectTargetAddressesFromIndexes(indexEvents, indexByAddress) const targetEventIds = collectPublicationIndexEventIds(indexEvents) if (import.meta.env.DEV) { @@ -565,11 +612,39 @@ export async function loadLibraryPublicationIndex( targetEventIds: targetEventIds.size }) } - const engagement = await fetchPublicationEngagementMaps( - relayUrls, - targetAddresses, - targetEventIds - ) + + let engagement: PublicationEngagementMaps + try { + engagement = await Promise.race([ + fetchPublicationEngagementMaps(relayUrls, targetAddresses, targetEventIds, { + httpOnly: true + }), + new Promise((resolve) => { + window.setTimeout( + () => + resolve({ + labelAddresses: new Set(), + labelEventIds: new Set(), + commentAddresses: new Set(), + highlightAddresses: new Set() + }), + ENGAGEMENT_FETCH_TIMEOUT_MS + ) + }) + ]) + } catch (e) { + if (import.meta.env.DEV) { + logger.warn('[Library] engagement fetch failed', { + message: e instanceof Error ? e.message : String(e) + }) + } + engagement = { + labelAddresses: new Set(), + labelEventIds: new Set(), + commentAddresses: new Set(), + highlightAddresses: new Set() + } + } if (import.meta.env.DEV) { logger.info('[Library] engagement maps built', { labels: engagement.labelAddresses.size + engagement.labelEventIds.size, @@ -580,7 +655,6 @@ export async function loadLibraryPublicationIndex( sessionCache = { relayKey: key, indexEvents, indexByAddress, engagement } - const topLevel = getTopLevelIndexEvents(indexEvents) const engaged = pickLibraryPublicationEntries(topLevel, indexByAddress, engagement) if (import.meta.env.DEV) { diff --git a/src/lib/publication-index.ts b/src/lib/publication-index.ts index cd7deed1..c88eb13d 100644 --- a/src/lib/publication-index.ts +++ b/src/lib/publication-index.ts @@ -16,7 +16,7 @@ export function eventTagAddress(event: Event): string | null { export function filterValidIndexEvents(events: Event[]): Event[] { return events.filter((event) => { if (event.kind !== ExtendedKind.PUBLICATION) return false - if (event.content != null && event.content.length > 0) return false + if ((event.content ?? '') !== '') return false const hasTitle = event.tags.some( (t) => (t[0] || '').trim().toLowerCase() === 'title' && t[1] ) diff --git a/src/pages/primary/LibraryPage/index.tsx b/src/pages/primary/LibraryPage/index.tsx index 58a13d7b..4c392166 100644 --- a/src/pages/primary/LibraryPage/index.tsx +++ b/src/pages/primary/LibraryPage/index.tsx @@ -21,6 +21,7 @@ const LibraryPage = forwardRef((_props, ref) => { showOnlyMine, setShowOnlyMine, loading, + engagementLoading, error, allIndexCount, topLevelCount, @@ -69,13 +70,15 @@ const LibraryPage = forwardRef((_props, ref) => { ) : null} {loading ? (

{t('Library loading')}

+ ) : engagementLoading ? ( +

{t('Library engagement loading')}

) : null} {statusLine ? (

{statusLine}

) : null}