From c843c4f85d924312a3b88cad19a2b2d84deb8f37 Mon Sep 17 00:00:00 2001 From: Silberengel Date: Mon, 16 Feb 2026 09:59:05 +0100 Subject: [PATCH] make filters more resilient and efficient --- config.yaml.example | 11 +- internal/generator/html.go | 46 +++--- internal/nostr/client.go | 284 ++++++++++++-------------------- internal/nostr/feed.go | 90 ++++------ internal/nostr/issues.go | 7 +- internal/nostr/profile.go | 80 +++++++-- internal/nostr/wiki.go | 317 ++++++++++++++---------------------- internal/server/handlers.go | 9 + static/css/main.css | 4 + 9 files changed, 359 insertions(+), 489 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index 961020f..8041945 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -19,14 +19,9 @@ seo: site_name: "GitCitadel" site_url: "https://gitcitadel.com" default_image: "/static/GitCitadel_Graphic_Landscape.png" -wiki_kinds: - - 30818 # Wiki pages -blog_kinds: - - 30041 # Blog articles -# article_kinds is deprecated, use wiki_kinds and blog_kinds instead -# article_kinds: -# - 30041 # Blog articles -# - 30818 # Wiki pages +wiki_kind: 30818 # Wiki pages +blog_kind: 30041 # Blog articles +longform_kind: 30023 # Markdown articles index_kind: 30040 # Index event kind (NKBIP-01) issue_kind: 1621 # Issue event kind repo_announcement_kind: 30617 # Repository announcement kind diff --git a/internal/generator/html.go b/internal/generator/html.go index f32002d..5ec247d 100644 --- a/internal/generator/html.go +++ b/internal/generator/html.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "strings" - "sync" "time" "gitcitadel-online/internal/asciidoc" @@ -193,42 +192,37 @@ func NewHTMLGenerator(templateDir string, linkBaseURL, siteName, siteURL, defaul }, nil } -// fetchProfilesBatch fetches profiles for multiple pubkeys in parallel +// fetchProfilesBatch fetches profiles for multiple pubkeys in a single batched query func (g *HTMLGenerator) fetchProfilesBatch(ctx context.Context, pubkeys []string) map[string]*nostr.Profile { if g.nostrClient == nil || len(pubkeys) == 0 { return make(map[string]*nostr.Profile) } - profiles := make(map[string]*nostr.Profile) - var mu sync.Mutex - var wg sync.WaitGroup + // Deduplicate pubkeys + pubkeySet := make(map[string]bool) + uniquePubkeys := make([]string, 0, len(pubkeys)) + for _, pk := range pubkeys { + if pk != "" && !pubkeySet[pk] { + pubkeySet[pk] = true + uniquePubkeys = append(uniquePubkeys, pk) + } + } + + if len(uniquePubkeys) == 0 { + return make(map[string]*nostr.Profile) + } // Create a context with timeout for profile fetching - profileCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + profileCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - for _, pubkey := range pubkeys { - if pubkey == "" { - continue - } - wg.Add(1) - go func(pk string) { - defer wg.Done() - // Convert pubkey to npub for fetching - npub, err := nostr.PubkeyToNpub(pk) - if err != nil { - return - } - profile, err := g.nostrClient.FetchProfile(profileCtx, npub) - if err == nil && profile != nil { - mu.Lock() - profiles[pk] = profile - mu.Unlock() - } - }(pubkey) + // Use batch fetch - single query for all profiles + profiles, err := g.nostrClient.FetchProfilesBatch(profileCtx, uniquePubkeys) + if err != nil { + // Log error but return empty map - profiles are optional + return make(map[string]*nostr.Profile) } - wg.Wait() return profiles } diff --git a/internal/nostr/client.go b/internal/nostr/client.go index 4074a9d..097f5a9 100644 --- a/internal/nostr/client.go +++ b/internal/nostr/client.go @@ -11,123 +11,94 @@ import ( "github.com/nbd-wtf/go-nostr" ) -// Client handles connections to Nostr relays with failover support +// Client handles connections to Nostr relays with failover support using go-nostr's SimplePool type Client struct { - primaryRelay string - fallbackRelay string - additionalFallback string - mu sync.RWMutex - lastError error + pool *nostr.SimplePool + relays []string + mu sync.RWMutex + ctx context.Context + requestSem chan struct{} // Semaphore to limit concurrent requests + maxConcurrent int // Maximum concurrent requests } // NewClient creates a new Nostr client with primary, fallback, and additional fallback relays +// Uses go-nostr's SimplePool for connection management and parallel queries +// Limits concurrent requests to prevent overwhelming relays (default: 5 concurrent requests) func NewClient(primaryRelay, fallbackRelay, additionalFallback string) *Client { + ctx := context.Background() + pool := nostr.NewSimplePool(ctx) + + relays := []string{} + if primaryRelay != "" { + relays = append(relays, primaryRelay) + } + if fallbackRelay != "" { + relays = append(relays, fallbackRelay) + } + if additionalFallback != "" { + relays = append(relays, additionalFallback) + } + + maxConcurrent := 5 // Limit to 5 concurrent requests to avoid overwhelming relays return &Client{ - primaryRelay: primaryRelay, - fallbackRelay: fallbackRelay, - additionalFallback: additionalFallback, + pool: pool, + relays: relays, + ctx: ctx, + requestSem: make(chan struct{}, maxConcurrent), + maxConcurrent: maxConcurrent, } } -// Connect connects to the relays (no-op for now, connections happen on query) +// Connect connects to the relays (no-op, SimplePool handles connections lazily) func (c *Client) Connect(ctx context.Context) error { - // Connections are established lazily when querying + // SimplePool handles connections lazily when querying return nil } -// ConnectToRelay connects to a single relay (exported for use by services) +// ConnectToRelay connects to a single relay using SimplePool (exported for use by services) func (c *Client) ConnectToRelay(ctx context.Context, url string) (*nostr.Relay, error) { - relay, err := nostr.RelayConnect(ctx, url) - if err != nil { - return nil, err - } - return relay, nil -} - -// connectToRelay connects to a single relay (deprecated, use ConnectToRelay) -func (c *Client) connectToRelay(ctx context.Context, url string) (*nostr.Relay, error) { - return c.ConnectToRelay(ctx, url) + // Use SimplePool's EnsureRelay which handles connection pooling and reuse + return c.pool.EnsureRelay(url) } -// FetchEvent fetches a single event by querying all three relays in parallel with 10-second timeout +// FetchEvent fetches a single event by querying all relays in parallel using SimplePool // Returns the newest event (highest created_at) if multiple events are found +// Rate-limited to prevent overwhelming relays func (c *Client) FetchEvent(ctx context.Context, filter nostr.Filter) (*nostr.Event, error) { - // Create context with 10-second timeout - queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - // Query all three relays in parallel - relays := []string{c.primaryRelay, c.fallbackRelay, c.additionalFallback} - type result struct { - relay string - events []*nostr.Event - err error - } + // Acquire semaphore to limit concurrent requests + c.requestSem <- struct{}{} + defer func() { <-c.requestSem }() - results := make(chan result, len(relays)) - var wg sync.WaitGroup - - for _, relayURL := range relays { - if relayURL == "" { - continue - } - wg.Add(1) - go func(url string) { - defer wg.Done() - logger.WithFields(map[string]interface{}{ - "relay": url, - "kinds": filter.Kinds, - "authors": filter.Authors, - "ids": filter.IDs, - "tags": filter.Tags, - }).Debug("Querying relay") - - relay, err := c.connectToRelay(queryCtx, url) - if err != nil { - logger.WithField("relay", url).Debugf("Connection failed: %v", err) - results <- result{relay: url, events: nil, err: err} - return - } - defer relay.Close() - - events, err := relay.QuerySync(queryCtx, filter) - if err != nil { - logger.WithField("relay", url).Debugf("Query failed: %v", err) - results <- result{relay: url, events: nil, err: err} - return - } + // Create context with 30-second timeout + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() - logger.WithFields(map[string]interface{}{ - "relay": url, - "events": len(events), - }).Debug("Relay returned events") - results <- result{relay: url, events: events, err: nil} - }(relayURL) - } + logger.WithFields(map[string]interface{}{ + "relays": c.relays, + "kinds": filter.Kinds, + "authors": filter.Authors, + "ids": filter.IDs, + "tags": filter.Tags, + }).Debug("Querying relays using SimplePool") - // Wait for all queries to complete - go func() { - wg.Wait() - close(results) - }() + // Use SimplePool's SubManyEose to query all relays in parallel + // It automatically handles connection pooling, failover, and deduplication + eventChan := c.pool.SubManyEose(queryCtx, c.relays, nostr.Filters{filter}) // Collect all events from all relays var allEvents []*nostr.Event - var lastErr error - for res := range results { - if res.err != nil { - lastErr = res.err - continue - } - if len(res.events) > 0 { - allEvents = append(allEvents, res.events...) + for incomingEvent := range eventChan { + if incomingEvent.Event != nil { + allEvents = append(allEvents, incomingEvent.Event) + logger.WithFields(map[string]interface{}{ + "relay": incomingEvent.Relay.URL, + "event_id": incomingEvent.Event.ID, + "created_at": incomingEvent.Event.CreatedAt, + }).Debug("Received event from relay") } } if len(allEvents) == 0 { - if lastErr != nil { - return nil, fmt.Errorf("event not found from any relay: %w", lastErr) - } return nil, fmt.Errorf("event not found from any relay") } @@ -146,89 +117,56 @@ func (c *Client) FetchEvent(ctx context.Context, filter nostr.Filter) (*nostr.Ev return newest, nil } -// FetchEvents fetches multiple events by querying all three relays in parallel with 10-second timeout +// FetchEvents fetches multiple events by querying all relays in parallel using SimplePool // Returns deduplicated events, keeping the newest version of each event +// Rate-limited to prevent overwhelming relays func (c *Client) FetchEvents(ctx context.Context, filter nostr.Filter) ([]*nostr.Event, error) { - // Create context with 10-second timeout - queryCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() + return c.FetchEventsBatch(ctx, []nostr.Filter{filter}) +} - // Query all three relays in parallel - relays := []string{c.primaryRelay, c.fallbackRelay, c.additionalFallback} - type result struct { - relay string - events []*nostr.Event - err error +// FetchEventsBatch fetches multiple events using multiple filters in a single batched query +// This is more efficient than calling FetchEvents multiple times +// Returns deduplicated events, keeping the newest version of each event +// Rate-limited to prevent overwhelming relays +func (c *Client) FetchEventsBatch(ctx context.Context, filters []nostr.Filter) ([]*nostr.Event, error) { + if len(filters) == 0 { + return nil, fmt.Errorf("no filters provided") } - results := make(chan result, len(relays)) - var wg sync.WaitGroup - - for _, relayURL := range relays { - if relayURL == "" { - continue - } - wg.Add(1) - go func(url string) { - defer wg.Done() - logger.WithFields(map[string]interface{}{ - "relay": url, - "kinds": filter.Kinds, - "authors": filter.Authors, - "ids": filter.IDs, - "tags": filter.Tags, - "limit": filter.Limit, - }).Debug("Querying relay") - - relay, err := c.connectToRelay(queryCtx, url) - if err != nil { - logger.WithField("relay", url).Debugf("Connection failed: %v", err) - results <- result{relay: url, events: nil, err: err} - return - } - defer relay.Close() + // Acquire semaphore to limit concurrent requests + c.requestSem <- struct{}{} + defer func() { <-c.requestSem }() - events, err := relay.QuerySync(queryCtx, filter) - if err != nil { - logger.WithField("relay", url).Debugf("Query failed: %v", err) - results <- result{relay: url, events: nil, err: err} - return - } + // Create context with 30-second timeout + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() - logger.WithFields(map[string]interface{}{ - "relay": url, - "events": len(events), - }).Debug("Relay returned events") - results <- result{relay: url, events: events, err: nil} - }(relayURL) - } + logger.WithFields(map[string]interface{}{ + "relays": c.relays, + "filters": len(filters), + }).Debug("Querying relays using SimplePool with batched filters") - // Wait for all queries to complete - go func() { - wg.Wait() - close(results) - }() + // Use SimplePool's SubManyEose to query all relays in parallel with all filters + // It automatically handles connection pooling, failover, and deduplication + eventChan := c.pool.SubManyEose(queryCtx, c.relays, nostr.Filters(filters)) // Collect all events from all relays, deduplicating by ID and keeping newest eventMap := make(map[string]*nostr.Event) - var lastErr error - for res := range results { - if res.err != nil { - lastErr = res.err - continue - } - for _, event := range res.events { - existing, exists := eventMap[event.ID] - if !exists || event.CreatedAt > existing.CreatedAt { - eventMap[event.ID] = event + for incomingEvent := range eventChan { + if incomingEvent.Event != nil { + existing, exists := eventMap[incomingEvent.Event.ID] + if !exists || incomingEvent.Event.CreatedAt > existing.CreatedAt { + eventMap[incomingEvent.Event.ID] = incomingEvent.Event + logger.WithFields(map[string]interface{}{ + "relay": incomingEvent.Relay.URL, + "event_id": incomingEvent.Event.ID, + "created_at": incomingEvent.Event.CreatedAt, + }).Debug("Received event from relay") } } } if len(eventMap) == 0 { - if lastErr != nil { - return nil, fmt.Errorf("no events found from any relay: %w", lastErr) - } return nil, fmt.Errorf("no events found from any relay") } @@ -259,36 +197,20 @@ func (c *Client) FetchEventsByKind(ctx context.Context, kind int, limit int) ([] return c.FetchEvents(ctx, filter) } -// Close closes all relay connections (no-op for lazy connections) +// Close closes all relay connections in the pool func (c *Client) Close() { - // Connections are closed after each query, so nothing to do here + // SimplePool manages connections, but we can close individual relays if needed + // The pool will handle cleanup when context is cancelled } -// GetLastError returns the last error encountered -func (c *Client) GetLastError() error { - c.mu.RLock() - defer c.mu.RUnlock() - return c.lastError -} - -// IsConnected checks if at least one relay is connected (always true for lazy connections) -func (c *Client) IsConnected() bool { - return true // We connect on-demand +// GetRelays returns all configured relay URLs +func (c *Client) GetRelays() []string { + return c.relays } -// GetRelays returns all configured relay URLs (primary, fallback, additional fallback) -func (c *Client) GetRelays() []string { - relays := []string{} - if c.primaryRelay != "" { - relays = append(relays, c.primaryRelay) - } - if c.fallbackRelay != "" { - relays = append(relays, c.fallbackRelay) - } - if c.additionalFallback != "" { - relays = append(relays, c.additionalFallback) - } - return relays +// GetPool returns the underlying SimplePool (for services that need direct access) +func (c *Client) GetPool() *nostr.SimplePool { + return c.pool } // HealthCheck performs a health check on the relays diff --git a/internal/nostr/feed.go b/internal/nostr/feed.go index e4fee13..e24462f 100644 --- a/internal/nostr/feed.go +++ b/internal/nostr/feed.go @@ -24,7 +24,7 @@ func NewFeedService(client *Client, feedKind int) *FeedService { } } -// FetchFeedItems fetches recent feed events from the configured feed relay only, with retries +// FetchFeedItems fetches recent feed events from the configured feed relay using SimplePool func (fs *FeedService) FetchFeedItems(ctx context.Context, feedRelay string, maxEvents int) ([]FeedItem, error) { if feedRelay == "" { return nil, fmt.Errorf("feed relay not configured") @@ -36,75 +36,43 @@ func (fs *FeedService) FetchFeedItems(ctx context.Context, feedRelay string, max } logFilter(filter, fmt.Sprintf("feed (kind %d)", fs.feedKind)) - const maxRetries = 3 - var lastErr error + // Create context with 30-second timeout + queryCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() - // Retry up to 3 times for the configured feed relay only - for attempt := 1; attempt <= maxRetries; attempt++ { - if attempt > 1 { - logger.WithFields(map[string]interface{}{ - "relay": feedRelay, - "attempt": attempt, - "max": maxRetries, - }).Debug("Retrying feed relay") - // Wait a bit before retrying (exponential backoff: 1s, 2s) - waitTime := time.Duration(attempt-1) * time.Second - time.Sleep(waitTime) - } - - // Create a context with timeout for this attempt - attemptCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - - // Connect to the relay - relayConn, err := fs.client.ConnectToRelay(attemptCtx, feedRelay) - if err != nil { - logger.WithFields(map[string]interface{}{ - "relay": feedRelay, - "attempt": attempt, - "max": maxRetries, - }).Debugf("Failed to connect to feed relay: %v", err) - lastErr = err - cancel() - continue // Try next attempt - } + logger.WithFields(map[string]interface{}{ + "relay": feedRelay, + "kind": fs.feedKind, + "limit": maxEvents, + }).Debug("Fetching feed items using SimplePool") - // Try to fetch events - events, err := relayConn.QuerySync(attemptCtx, filter) - relayConn.Close() - cancel() + // Use the client's pool to query the feed relay + // SimplePool handles connection pooling and reuse automatically + eventChan := fs.client.GetPool().SubManyEose(queryCtx, []string{feedRelay}, nostr.Filters{filter}) - if err != nil { - logger.WithFields(map[string]interface{}{ - "relay": feedRelay, - "attempt": attempt, - "max": maxRetries, - }).Debugf("Failed to fetch feed events: %v", err) - lastErr = err - continue // Try next attempt - } - - // Success! Convert events to feed items - items := make([]FeedItem, 0, len(events)) - for _, event := range events { + // Collect events and convert to feed items + items := make([]FeedItem, 0, maxEvents) + for incomingEvent := range eventChan { + if incomingEvent.Event != nil { item := FeedItem{ - Author: event.PubKey, - Content: event.Content, - Time: time.Unix(int64(event.CreatedAt), 0), - Link: fmt.Sprintf("https://alexandria.gitcitadel.eu/events?id=nevent1%s", event.ID), + Author: incomingEvent.Event.PubKey, + Content: incomingEvent.Event.Content, + Time: time.Unix(int64(incomingEvent.Event.CreatedAt), 0), + Link: fmt.Sprintf("https://alexandria.gitcitadel.eu/events?id=nevent1%s", incomingEvent.Event.ID), } items = append(items, item) + logger.WithFields(map[string]interface{}{ + "relay": incomingEvent.Relay.URL, + "event_id": incomingEvent.Event.ID, + }).Debug("Received feed event") } - - logger.WithFields(map[string]interface{}{ - "relay": feedRelay, - "items": len(items), - "attempt": attempt, - }).Info("Successfully fetched feed items") - return items, nil } - // All retries failed - return nil, fmt.Errorf("failed to fetch feed from %s after %d retries (last error: %w)", feedRelay, maxRetries, lastErr) + logger.WithFields(map[string]interface{}{ + "relay": feedRelay, + "items": len(items), + }).Info("Successfully fetched feed items") + return items, nil } // FeedItem represents a feed item diff --git a/internal/nostr/issues.go b/internal/nostr/issues.go index a079158..90a52d2 100644 --- a/internal/nostr/issues.go +++ b/internal/nostr/issues.go @@ -168,7 +168,10 @@ func (s *IssueService) PublishIssue(ctx context.Context, repoAnnouncement *RepoA relays := repoAnnouncement.Relays if len(relays) == 0 { // Fallback to default relays if none specified - relays = []string{s.client.primaryRelay, s.client.fallbackRelay} + clientRelays := s.client.GetRelays() + if len(clientRelays) > 0 { + relays = clientRelays + } } // Publish to relays @@ -217,7 +220,7 @@ func (s *IssueService) PublishSignedIssue(ctx context.Context, signedEvent *nost // Determine which relays to publish to // Try to extract relays from the event tags or use defaults - relays := []string{s.client.primaryRelay, s.client.fallbackRelay} + relays := s.client.GetRelays() // Look for relay hints in tags for _, tag := range signedEvent.Tags { diff --git a/internal/nostr/profile.go b/internal/nostr/profile.go index 3a444a6..60b3b96 100644 --- a/internal/nostr/profile.go +++ b/internal/nostr/profile.go @@ -73,31 +73,81 @@ func (c *Client) FetchProfile(ctx context.Context, npub string) (*Profile, error return nil, fmt.Errorf("failed to parse npub: unexpected type") } - // Create filter for kind 0 profile event + profiles, err := c.FetchProfilesBatch(ctx, []string{pubkey}) + if err != nil { + return nil, err + } + + profile, exists := profiles[pubkey] + if !exists { + return nil, fmt.Errorf("profile not found for pubkey") + } + + return profile, nil +} + +// FetchProfilesBatch fetches kind 0 profile events for multiple pubkeys in a single query +// Returns a map of pubkey -> Profile +func (c *Client) FetchProfilesBatch(ctx context.Context, pubkeys []string) (map[string]*Profile, error) { + if len(pubkeys) == 0 { + return make(map[string]*Profile), nil + } + + // Deduplicate pubkeys + pubkeySet := make(map[string]bool) + uniquePubkeys := make([]string, 0, len(pubkeys)) + for _, pk := range pubkeys { + if pk != "" && !pubkeySet[pk] { + pubkeySet[pk] = true + uniquePubkeys = append(uniquePubkeys, pk) + } + } + + if len(uniquePubkeys) == 0 { + return make(map[string]*Profile), nil + } + + // Query ALL kind 0 events for these authors in one batch filter := nostr.Filter{ Kinds: []int{0}, - Authors: []string{pubkey}, - Limit: 1, + Authors: uniquePubkeys, + Limit: len(uniquePubkeys), // One profile per author } logger.WithFields(map[string]interface{}{ - "npub": npub, - "pubkey": pubkey, - }).Debug("Fetching profile") + "pubkeys": len(uniquePubkeys), + }).Debug("Batch fetching profiles") - // Fetch the event - event, err := c.FetchEvent(ctx, filter) + // Fetch all profile events + events, err := c.FetchEvents(ctx, filter) if err != nil { - return nil, fmt.Errorf("failed to fetch profile event: %w", err) + return nil, fmt.Errorf("failed to fetch profile events: %w", err) + } + + // Parse profiles and map by pubkey + profiles := make(map[string]*Profile) + for _, event := range events { + profile, err := ParseProfile(event) + if err != nil { + logger.WithFields(map[string]interface{}{ + "pubkey": event.PubKey, + "event_id": event.ID, + }).Warnf("Error parsing profile: %v", err) + continue + } + // Keep the newest profile if we have multiple for the same pubkey + existing, exists := profiles[event.PubKey] + if !exists || event.CreatedAt > existing.Event.CreatedAt { + profiles[event.PubKey] = profile + } } - // Parse the profile - profile, err := ParseProfile(event) - if err != nil { - return nil, fmt.Errorf("failed to parse profile: %w", err) - } + logger.WithFields(map[string]interface{}{ + "requested": len(uniquePubkeys), + "fetched": len(profiles), + }).Debug("Batch profile fetch completed") - return profile, nil + return profiles, nil } // ParseProfile parses a kind 0 profile event diff --git a/internal/nostr/wiki.go b/internal/nostr/wiki.go index b994e91..7fb206f 100644 --- a/internal/nostr/wiki.go +++ b/internal/nostr/wiki.go @@ -71,127 +71,90 @@ func (ws *WikiService) FetchWikiIndex(ctx context.Context, naddrStr string) (*In } // FetchWikiEvents fetches all wiki events referenced in an index +// Queries by kind only, then filters locally and batch-fetches profiles func (ws *WikiService) FetchWikiEvents(ctx context.Context, index *IndexEvent) ([]*WikiEvent, error) { - var wikiEvents []*WikiEvent - + // Build a map of expected items (kind:pubkey:dtag) for fast lookup + expectedItems := make(map[string]IndexItem) for _, item := range index.Items { - if item.Kind != ws.wikiKind { - continue // Skip non-wiki items + if item.Kind == ws.wikiKind { + key := fmt.Sprintf("%d:%s:%s", item.Kind, item.Pubkey, item.DTag) + expectedItems[key] = item } + } - // Create filter for this wiki event - filter := nostr.Filter{ - Kinds: []int{ws.wikiKind}, - Authors: []string{item.Pubkey}, - Tags: map[string][]string{ - "d": {item.DTag}, - }, - } + if len(expectedItems) == 0 { + return []*WikiEvent{}, nil + } - // If event ID is specified, use it for more reliable fetching - if item.EventID != "" { - filter.IDs = []string{item.EventID} - logger.WithFields(map[string]interface{}{ - "event_id": item.EventID[:16] + "...", - "dtag": item.DTag, - }).Debug("Using event ID for wiki event") - } + // Query ALL events of this kind - simple query by kind only + filter := nostr.Filter{ + Kinds: []int{ws.wikiKind}, + Limit: 10000, // Reasonable limit + } - logFilter(filter, fmt.Sprintf("wiki event %s", item.DTag)) + logger.WithFields(map[string]interface{}{ + "kind": ws.wikiKind, + "items": len(expectedItems), + }).Debug("Querying all events of kind from relays") - // Use relay hint if available, otherwise use default client relays - var event *nostr.Event - var err error - if item.RelayHint != "" { - // Connect to the relay hint - logger.WithFields(map[string]interface{}{ - "relay_hint": item.RelayHint, - "dtag": item.DTag, - }).Debug("Trying relay hint") - relay, relayErr := ws.client.ConnectToRelay(ctx, item.RelayHint) - if relayErr == nil { - events, queryErr := relay.QuerySync(ctx, filter) - relay.Close() - if queryErr == nil { - if len(events) > 0 { - event = events[0] - logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, - "relay_hint": item.RelayHint, - }).Debug("Successfully fetched from relay hint") - } else { - logger.WithFields(map[string]interface{}{ - "relay_hint": item.RelayHint, - "dtag": item.DTag, - "has_event_id": item.EventID != "", - }).Debug("Relay hint returned 0 events") - } - } else { - logger.WithFields(map[string]interface{}{ - "relay_hint": item.RelayHint, - "dtag": item.DTag, - }).Debugf("Error querying relay hint: %v", queryErr) - } - } else { - logger.WithFields(map[string]interface{}{ - "relay_hint": item.RelayHint, - "dtag": item.DTag, - }).Debugf("Error connecting to relay hint: %v", relayErr) + // Fetch all events of this kind + allEvents, err := ws.client.FetchEvents(ctx, filter) + if err != nil { + logger.WithField("error", err).Warn("Failed to fetch events by kind") + return nil, err + } + + logger.WithFields(map[string]interface{}{ + "fetched": len(allEvents), + "expected": len(expectedItems), + }).Debug("Fetched events, filtering locally") + + // Filter events locally by matching against index items + eventMap := make(map[string]*nostr.Event) // Map by kind:pubkey:dtag + for _, event := range allEvents { + // Extract d-tag from event + var dTag string + for _, tag := range event.Tags { + if len(tag) > 0 && tag[0] == "d" && len(tag) > 1 { + dTag = tag[1] + break } } - // Fallback to default client relays if relay hint failed or wasn't provided - // Client now queries all three relays automatically - if event == nil { - logger.WithField("dtag", item.DTag).Debug("Querying all relays") - event, err = ws.client.FetchEvent(ctx, filter) - if err != nil { - // If still not found and event ID was specified, try without event ID to get latest replaceable event - if item.EventID != "" { - logger.WithField("dtag", item.DTag).Debug("Trying without event ID to get latest replaceable event") - filterWithoutID := nostr.Filter{ - Kinds: []int{ws.wikiKind}, - Authors: []string{item.Pubkey}, - Tags: map[string][]string{ - "d": {item.DTag}, - }, - } - logFilter(filterWithoutID, fmt.Sprintf("wiki event %s (without event ID)", item.DTag)) - event, err = ws.client.FetchEvent(ctx, filterWithoutID) - if err == nil { - logger.WithField("dtag", item.DTag).Debug("Successfully fetched latest from all relays") - } - } - } else { - logger.WithField("dtag", item.DTag).Debug("Successfully fetched from all relays") - } + if dTag == "" { + continue // Skip events without d-tag + } - if event == nil { - // Log error but continue with other events - logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, - "pubkey": item.Pubkey, - "has_event_id": item.EventID != "", - }).Warnf("Error fetching wiki event: %v", err) - continue + key := fmt.Sprintf("%d:%s:%s", event.Kind, event.PubKey, dTag) + if _, expected := expectedItems[key]; expected { + // Keep the newest version if we have multiple + existing, exists := eventMap[key] + if !exists || event.CreatedAt > existing.CreatedAt { + eventMap[key] = event } } + } - // Parse the wiki event + // Convert matched events to wiki events + var wikiEvents []*WikiEvent + for key, event := range eventMap { wiki, err := ParseWikiEvent(event, ws.wikiKind) if err != nil { logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, + "key": key, }).Warnf("Error parsing wiki event: %v", err) continue } - wikiEvents = append(wikiEvents, wiki) } + logger.WithFields(map[string]interface{}{ + "matched": len(wikiEvents), + "expected": len(expectedItems), + }).Debug("Matched wiki events") + if len(wikiEvents) == 0 && len(index.Items) > 0 { - logger.WithField("items", len(index.Items)).Warn("No wiki events could be fetched from index items") - // Return empty slice instead of error to allow landing page generation + logger.WithField("items", len(index.Items)).Warn("No wiki events matched from fetched events") } return wikiEvents, nil @@ -204,6 +167,7 @@ func (ws *WikiService) GetBlogKind() int { // FetchIndexEvents fetches all events of a specific kind referenced in an index // Only supports article kinds configured in the service +// Queries by kind only, then filters locally func (ws *WikiService) FetchIndexEvents(ctx context.Context, index *IndexEvent, targetKind int) ([]*nostr.Event, error) { // Check if the target kind is in the allowed article kinds allowed := false @@ -217,124 +181,85 @@ func (ws *WikiService) FetchIndexEvents(ctx context.Context, index *IndexEvent, return nil, fmt.Errorf("unsupported event kind: %d (only %v are supported)", targetKind, ws.articleKinds) } - var events []*nostr.Event - + // Build a map of expected items (kind:pubkey:dtag) for fast lookup + expectedItems := make(map[string]IndexItem) for _, item := range index.Items { - // Only process items of the target kind - if item.Kind != targetKind { - continue // Skip items that don't match the target kind + if item.Kind == targetKind { + key := fmt.Sprintf("%d:%s:%s", item.Kind, item.Pubkey, item.DTag) + expectedItems[key] = item } + } - // Create filter for this event - filter := nostr.Filter{ - Kinds: []int{targetKind}, - Authors: []string{item.Pubkey}, - Tags: map[string][]string{ - "d": {item.DTag}, - }, - } + if len(expectedItems) == 0 { + return []*nostr.Event{}, nil + } - // If event ID is specified, use it for more reliable fetching - if item.EventID != "" { - filter.IDs = []string{item.EventID} - } + // Query ALL events of this kind - simple query by kind only + filter := nostr.Filter{ + Kinds: []int{targetKind}, + Limit: 10000, // Reasonable limit + } + + logger.WithFields(map[string]interface{}{ + "kind": targetKind, + "items": len(expectedItems), + }).Debug("Querying all events of kind from relays") - logFilter(filter, fmt.Sprintf("index event kind %d %s", targetKind, item.DTag)) - - // Use relay hint if available, otherwise use default client relays - var event *nostr.Event - var err error - if item.RelayHint != "" { - // Connect to the relay hint - relay, relayErr := ws.client.ConnectToRelay(ctx, item.RelayHint) - if relayErr == nil { - relayEvents, queryErr := relay.QuerySync(ctx, filter) - relay.Close() - if queryErr == nil { - if len(relayEvents) > 0 { - event = relayEvents[0] - } else { - logger.WithFields(map[string]interface{}{ - "relay_hint": item.RelayHint, - "dtag": item.DTag, - "kind": targetKind, - "has_event_id": item.EventID != "", - }).Debug("Relay hint returned 0 events") - } - } else { - logger.WithFields(map[string]interface{}{ - "relay_hint": item.RelayHint, - "dtag": item.DTag, - "kind": targetKind, - }).Debugf("Error querying relay hint: %v", queryErr) - } - } else { - logger.WithFields(map[string]interface{}{ - "relay_hint": item.RelayHint, - "dtag": item.DTag, - "kind": targetKind, - }).Debugf("Error connecting to relay hint: %v", relayErr) + // Fetch all events of this kind + allEvents, err := ws.client.FetchEvents(ctx, filter) + if err != nil { + logger.WithField("error", err).Warn("Failed to fetch events by kind") + return nil, err + } + + logger.WithFields(map[string]interface{}{ + "fetched": len(allEvents), + "expected": len(expectedItems), + }).Debug("Fetched events, filtering locally") + + // Filter events locally by matching against index items + eventMap := make(map[string]*nostr.Event) // Map by kind:pubkey:dtag + for _, event := range allEvents { + // Extract d-tag from event + var dTag string + for _, tag := range event.Tags { + if len(tag) > 0 && tag[0] == "d" && len(tag) > 1 { + dTag = tag[1] + break } } - // Fallback to default client relays if relay hint failed or wasn't provided - // Client now queries all three relays automatically - if event == nil { - logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, - "kind": targetKind, - }).Debug("Querying all relays") - event, err = ws.client.FetchEvent(ctx, filter) - if err != nil { - // If still not found and event ID was specified, try without event ID to get latest replaceable event - if item.EventID != "" { - logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, - "kind": targetKind, - }).Debug("Trying without event ID to get latest replaceable event") - filterWithoutID := nostr.Filter{ - Kinds: []int{targetKind}, - Authors: []string{item.Pubkey}, - Tags: map[string][]string{ - "d": {item.DTag}, - }, - } - logFilter(filterWithoutID, fmt.Sprintf("index event kind %d %s (without event ID)", targetKind, item.DTag)) - event, err = ws.client.FetchEvent(ctx, filterWithoutID) - if err == nil { - logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, - "kind": targetKind, - }).Debug("Successfully fetched latest from all relays") - } - } - } else { - logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, - "kind": targetKind, - }).Debug("Successfully fetched from all relays") - } + if dTag == "" { + continue // Skip events without d-tag + } - if event == nil { - // Log error but continue with other events - logger.WithFields(map[string]interface{}{ - "dtag": item.DTag, - "kind": targetKind, - "pubkey": item.Pubkey, - "has_event_id": item.EventID != "", - }).Warnf("Error fetching event: %v", err) - continue + key := fmt.Sprintf("%d:%s:%s", event.Kind, event.PubKey, dTag) + if _, expected := expectedItems[key]; expected { + // Keep the newest version if we have multiple + existing, exists := eventMap[key] + if !exists || event.CreatedAt > existing.CreatedAt { + eventMap[key] = event } } + } + // Convert to result slice + events := make([]*nostr.Event, 0, len(eventMap)) + for _, event := range eventMap { events = append(events, event) } + logger.WithFields(map[string]interface{}{ + "matched": len(events), + "expected": len(expectedItems), + "kind": targetKind, + }).Debug("Matched index events") + if len(events) == 0 && len(index.Items) > 0 { logger.WithFields(map[string]interface{}{ "kind": targetKind, "items": len(index.Items), - }).Warn("No events could be fetched from index items") + }).Warn("No events matched from fetched events") } return events, nil diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 37d067a..dbf1906 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -21,6 +21,9 @@ func (s *Server) setupRoutes(mux *http.ServeMux) { // Static files mux.HandleFunc("/static/", s.handleStatic) + // Favicon + mux.HandleFunc("/favicon.ico", s.handleFavicon) + // Main routes mux.HandleFunc("/", s.handleLanding) mux.HandleFunc("/wiki/", s.handleWiki) @@ -283,6 +286,12 @@ func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request) { http.StripPrefix("/static/", http.FileServer(http.Dir("./static"))).ServeHTTP(w, r) } +// handleFavicon serves the favicon +func (s *Server) handleFavicon(w http.ResponseWriter, r *http.Request) { + // Serve the SVG icon as favicon + http.ServeFile(w, r, "./static/GitCitadel_Icon_Black.svg") +} + // handleHealth handles health check requests func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { if s.cache.Size() == 0 { diff --git a/static/css/main.css b/static/css/main.css index 2482939..b1c8ebc 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -77,12 +77,16 @@ header { box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); margin: 0; padding: 0; + will-change: transform; /* Optimize for fixed positioning */ } .navbar { padding: 1rem 0; width: 100%; margin: 0; + min-height: 64px; /* Ensure minimum height for header */ + display: flex; + align-items: center; } .nav-container {