fixes memory problem with many pubkeys in query

app/handle-req.go

@@ -129,6 +129,87 @@ func (l *Listener) HandleReq(msg []byte) (err error) {
 					)
 				},
 			)
+
+			// Process large author lists by breaking them into chunks
+			if f.Authors != nil && f.Authors.Len() > 50 {
+				log.W.F("REQ %s: breaking down large author list (%d authors) into chunks", env.Subscription, f.Authors.Len())
+
+				// Calculate chunk size based on kinds to avoid OOM
+				chunkSize := 50
+				if f.Kinds != nil && f.Kinds.Len() > 0 {
+					// Reduce chunk size if there are multiple kinds to prevent too many index ranges
+					chunkSize = 50 / f.Kinds.Len()
+					if chunkSize < 10 {
+						chunkSize = 10 // Minimum chunk size
+					}
+				}
+
+				// Process authors in chunks
+				for i := 0; i < f.Authors.Len(); i += chunkSize {
+					end := i + chunkSize
+					if end > f.Authors.Len() {
+						end = f.Authors.Len()
+					}
+
+					// Create a chunk filter
+					chunkAuthors := tag.NewFromBytesSlice(f.Authors.T[i:end]...)
+					chunkFilter := &filter.F{
+						Kinds:   f.Kinds,
+						Authors: chunkAuthors,
+						Ids:     f.Ids,
+						Tags:    f.Tags,
+						Since:   f.Since,
+						Until:   f.Until,
+						Limit:   f.Limit,
+						Search:  f.Search,
+					}
+
+					log.T.F("REQ %s: processing chunk %d-%d of %d authors", env.Subscription, i+1, end, f.Authors.Len())
+
+					// Process this chunk
+					var chunkEvents event.S
+					showAllVersions := false
+					if chunkFilter.Tags != nil {
+						if showAllTag := chunkFilter.Tags.GetFirst([]byte("show_all_versions")); showAllTag != nil {
+							if string(showAllTag.Value()) == "true" {
+								showAllVersions = true
+							}
+						}
+					}
+
+					if showAllVersions {
+						if chunkEvents, err = l.QueryAllVersions(queryCtx, chunkFilter); chk.E(err) {
+							if errors.Is(err, badger.ErrDBClosed) {
+								return
+							}
+							log.E.F("QueryAllVersions failed for chunk filter: %v", err)
+							err = nil
+							continue
+						}
+					} else {
+						if chunkEvents, err = l.QueryEvents(queryCtx, chunkFilter); chk.E(err) {
+							if errors.Is(err, badger.ErrDBClosed) {
+								return
+							}
+							log.E.F("QueryEvents failed for chunk filter: %v", err)
+							err = nil
+							continue
+						}
+					}
+
+					// Add chunk results to overall results
+					allEvents = append(allEvents, chunkEvents...)
+
+					// Check if we've hit the limit
+					if f.Limit != nil && len(allEvents) >= int(*f.Limit) {
+						log.T.F("REQ %s: reached limit of %d events, stopping chunk processing", env.Subscription, *f.Limit)
+						break
+					}
+				}
+
+				// Skip the normal processing since we handled it in chunks
+				continue
+			}
 		}
 		if f != nil && pointers.Present(f.Limit) {
 			if *f.Limit == 0 {
@@ -136,13 +217,35 @@ func (l *Listener) HandleReq(msg []byte) (err error) {
 			}
 		}
 		var filterEvents event.S
-		if filterEvents, err = l.QueryEvents(queryCtx, f); chk.E(err) {
+		// Check if the filter has the special "show_all_versions" tag
-			if errors.Is(err, badger.ErrDBClosed) {
+		showAllVersions := false
-				return
+		if f.Tags != nil {
+			if showAllTag := f.Tags.GetFirst([]byte("show_all_versions")); showAllTag != nil {
+				if string(showAllTag.Value()) == "true" {
+					showAllVersions = true
+					log.T.F("REQ %s: detected show_all_versions tag, using QueryAllVersions", env.Subscription)
+				}
+			}
+		}
+
+		if showAllVersions {
+			if filterEvents, err = l.QueryAllVersions(queryCtx, f); chk.E(err) {
+				if errors.Is(err, badger.ErrDBClosed) {
+					return
+				}
+				log.E.F("QueryAllVersions failed for filter: %v", err)
+				err = nil
+				continue
+			}
+		} else {
+			if filterEvents, err = l.QueryEvents(queryCtx, f); chk.E(err) {
+				if errors.Is(err, badger.ErrDBClosed) {
+					return
+				}
+				log.E.F("QueryEvents failed for filter: %v", err)
+				err = nil
+				continue
 			}
-			log.E.F("QueryEvents failed for filter: %v", err)
-			err = nil
-			continue
 		}
 		// Append events from this filter to the overall collection
 		allEvents = append(allEvents, filterEvents...)
@@ -275,10 +378,28 @@ privCheck:
 		events = policyFilteredEvents
 	}
 
+	// Deduplicate events (in case chunk processing returned duplicates)
+	if len(allEvents) > 0 {
+		seen := make(map[string]struct{})
+		var deduplicatedEvents event.S
+		originalCount := len(allEvents)
+		for _, ev := range allEvents {
+			eventID := hexenc.Enc(ev.ID)
+			if _, exists := seen[eventID]; !exists {
+				seen[eventID] = struct{}{}
+				deduplicatedEvents = append(deduplicatedEvents, ev)
+			}
+		}
+		allEvents = deduplicatedEvents
+		if originalCount != len(allEvents) {
+			log.T.F("REQ %s: deduplicated %d events to %d unique events", env.Subscription, originalCount, len(allEvents))
+		}
+	}
+
 	// Apply managed ACL filtering for read access if managed ACL is active
 	if acl.Registry.Active.Load() == "managed" {
 		var aclFilteredEvents event.S
-		for _, ev := range events {
+		for _, ev := range allEvents {
 			// Check if event is banned
 			eventID := hex.EncodeToString(ev.ID)
 			if banned, err := l.getManagedACL().IsEventBanned(eventID); err == nil && banned {
@@ -304,11 +425,11 @@ privCheck:
 
 			aclFilteredEvents = append(aclFilteredEvents, ev)
 		}
-		events = aclFilteredEvents
+		allEvents = aclFilteredEvents
 	}
 
 	seen := make(map[string]struct{})
-	for _, ev := range events {
+	for _, ev := range allEvents {
 		log.T.C(
 			func() string {
 				return fmt.Sprintf(

app/listener.go

@@ -10,6 +10,8 @@ import (
 	"lol.mleku.dev/log"
 	"next.orly.dev/pkg/acl"
 	"next.orly.dev/pkg/database"
+	"next.orly.dev/pkg/encoders/event"
+	"next.orly.dev/pkg/encoders/filter"
 	"next.orly.dev/pkg/utils/atomic"
 )
 
@@ -119,3 +121,13 @@ func (l *Listener) getManagedACL() *database.ManagedACL {
 	}
 	return nil
 }
+
+// QueryEvents queries events using the database QueryEvents method
+func (l *Listener) QueryEvents(ctx context.Context, f *filter.F) (event.S, error) {
+	return l.D.QueryEvents(ctx, f)
+}
+
+// QueryAllVersions queries events using the database QueryAllVersions method
+func (l *Listener) QueryAllVersions(ctx context.Context, f *filter.F) (event.S, error) {
+	return l.D.QueryAllVersions(ctx, f)
+}

app/web/src/App.svelte

@@ -75,6 +75,59 @@
     // Compose tab state
     let composeEventJson = "";
 
+    // Recovery tab state
+    let recoverySelectedKind = null;
+    let recoveryCustomKind = "";
+    let recoveryEvents = [];
+    let isLoadingRecovery = false;
+    let recoveryHasMore = true;
+    let recoveryOldestTimestamp = null;
+    let recoveryNewestTimestamp = null;
+
+    // Replaceable kinds for the recovery dropdown
+    const replaceableKinds = [
+        { value: 0, label: "Profile Metadata (0)" },
+        { value: 3, label: "Follow List (3)" },
+        { value: 10000, label: "Relay List Metadata (10000)" },
+        { value: 10001, label: "Mute List (10001)" },
+        { value: 10002, label: "Pin List (10002)" },
+        { value: 10003, label: "Bookmark List (10003)" },
+        { value: 10004, label: "Communities List (10004)" },
+        { value: 10005, label: "Public Chats List (10005)" },
+        { value: 10006, label: "Blocked Relays List (10006)" },
+        { value: 10007, label: "Search Relays List (10007)" },
+        { value: 10015, label: "Interests List (10015)" },
+        { value: 10030, label: "Emoji Sets (10030)" },
+        { value: 30000, label: "Categorized People List (30000)" },
+        { value: 30001, label: "Categorized Bookmark List (30001)" },
+        { value: 30002, label: "Relay Sets (30002)" },
+        { value: 30003, label: "Bookmark Sets (30003)" },
+        { value: 30004, label: "Curation Sets (30004)" },
+        { value: 30008, label: "Profile Badges (30008)" },
+        { value: 30009, label: "Badge Definition (30009)" },
+        { value: 30015, label: "Interest Sets (30015)" },
+        { value: 30017, label: "Stall Definition (30017)" },
+        { value: 30018, label: "Product Definition (30018)" },
+        { value: 30019, label: "Marketplace UI/UX (30019)" },
+        { value: 30020, label: "Product Sold as Auction (30020)" },
+        { value: 30023, label: "Article (30023)" },
+        { value: 30024, label: "Draft Long-form Content (30024)" },
+        { value: 30030, label: "Emoji Sets (30030)" },
+        { value: 30078, label: "Application Specific Data (30078)" },
+        { value: 30311, label: "Live Event (30311)" },
+        { value: 30315, label: "User Statuses (30315)" },
+        { value: 30402, label: "Classified Listing (30402)" },
+        { value: 30403, label: "Draft Classified Listing (30403)" },
+        { value: 31922, label: "Date-based Calendar Event (31922)" },
+        { value: 31923, label: "Time-based Calendar Event (31923)" },
+        { value: 31924, label: "Calendar (31924)" },
+        { value: 31925, label: "Calendar Event RSVP (31925)" },
+        { value: 31989, label: "Handler Recommendation (31989)" },
+        { value: 31990, label: "Handler Information (31990)" },
+        { value: 32123, label: "WaveLake Track (32123)" },
+        { value: 34550, label: "Community Definition (34550)" },
+    ];
+
     // Kind name mapping based on repository kind definitions
     const kindNames = {
         0: "ProfileMetadata",
@@ -595,6 +648,137 @@
             .replace(/'/g, "'");
     }
 
+    // Recovery tab functions
+    async function loadRecoveryEvents() {
+        const kindToUse = recoveryCustomKind
+            ? parseInt(recoveryCustomKind)
+            : recoverySelectedKind;
+        if (!kindToUse || !isLoggedIn) return;
+
+        console.log(
+            "Loading recovery events for kind:",
+            kindToUse,
+            "user:",
+            userPubkey,
+        );
+        isLoadingRecovery = true;
+        try {
+            // Always fetch all versions for replaceable kinds, then filter in the UI
+            const filters = {
+                kinds: [kindToUse],
+                authors: [userPubkey],
+                limit: 100,
+                tags: [["show_all_versions", "true"]], // Always get all versions
+            };
+
+            if (recoveryOldestTimestamp) {
+                filters.until = recoveryOldestTimestamp;
+            }
+
+            console.log("Recovery filters:", filters);
+            const events = await fetchAllEvents(filters);
+            console.log("Recovery events received:", events.length);
+            console.log(
+                "Recovery events kinds:",
+                events.map((e) => e.kind),
+            );
+
+            if (recoveryOldestTimestamp) {
+                // Append to existing events
+                recoveryEvents = [...recoveryEvents, ...events];
+            } else {
+                // Replace events
+                recoveryEvents = events;
+            }
+
+            if (events.length > 0) {
+                recoveryOldestTimestamp = Math.min(
+                    ...events.map((e) => e.created_at),
+                );
+                recoveryHasMore = events.length === 100;
+            } else {
+                recoveryHasMore = false;
+            }
+        } catch (error) {
+            console.error("Failed to load recovery events:", error);
+        } finally {
+            isLoadingRecovery = false;
+        }
+    }
+
+    async function repostEvent(event) {
+        if (!isLoggedIn || !userSigner) {
+            alert("Please log in to repost events");
+            return;
+        }
+
+        try {
+            // Create a new event with the same content but current timestamp
+            const newEvent = {
+                kind: event.kind,
+                content: event.content,
+                tags: event.tags,
+                created_at: Math.floor(Date.now() / 1000),
+                pubkey: userPubkey,
+            };
+
+            // Sign and publish the event
+            const result = await publishEventWithAuth(
+                DEFAULT_RELAYS[0],
+                newEvent,
+                userSigner,
+                userPubkey,
+            );
+
+            if (result.success) {
+                alert("Event reposted successfully!");
+                // Reload the recovery events to show the new current version
+                recoveryEvents = [];
+                recoveryOldestTimestamp = null;
+                await loadRecoveryEvents();
+            } else {
+                alert("Failed to repost event");
+            }
+        } catch (error) {
+            console.error("Failed to repost event:", error);
+            alert("Failed to repost event: " + error.message);
+        }
+    }
+
+    function selectRecoveryKind(kind) {
+        recoverySelectedKind = kind;
+        recoveryCustomKind = ""; // Clear custom kind when selecting from dropdown
+        recoveryEvents = [];
+        recoveryOldestTimestamp = null;
+        recoveryHasMore = true;
+        loadRecoveryEvents();
+    }
+
+    function handleCustomKindInput() {
+        if (recoveryCustomKind) {
+            recoverySelectedKind = null; // Clear dropdown selection when using custom
+            recoveryEvents = [];
+            recoveryOldestTimestamp = null;
+            recoveryHasMore = true;
+            loadRecoveryEvents();
+        }
+    }
+
+    function isCurrentVersion(event) {
+        // Find all events with the same kind and pubkey
+        const sameKindEvents = recoveryEvents.filter(
+            (e) => e.kind === event.kind && e.pubkey === event.pubkey,
+        );
+
+        // Check if this event has the highest timestamp
+        const maxTimestamp = Math.max(
+            ...sameKindEvents.map((e) => e.created_at),
+        );
+        return event.created_at === maxTimestamp;
+    }
+
+    // Always show all versions - no filtering needed
+
     $: aboutHtml = userProfile?.about
         ? escapeHtml(userProfile.about).replace(/\n{2,}/g, "<br>")
         : "";
@@ -1073,6 +1257,7 @@
         { id: "import", icon: "💾", label: "Import", requiresAdmin: true },
         { id: "events", icon: "📡", label: "Events" },
         { id: "compose", icon: "✏️", label: "Compose" },
+        { id: "recovery", icon: "🔄", label: "Recovery" },
         {
             id: "managed-acl",
             icon: "🛡️",
@@ -2584,6 +2769,150 @@
                     </div>
                 {/if}
             </div>
+        {:else if selectedTab === "recovery"}
+            <div class="recovery-tab">
+                <div class="recovery-header">
+                    <h2>🔄 Event Recovery</h2>
+                    <p>Search and recover old versions of replaceable events</p>
+                </div>
+
+                <div class="recovery-controls">
+                    <div class="kind-selector">
+                        <label for="recovery-kind">Select Event Kind:</label>
+                        <select
+                            id="recovery-kind"
+                            bind:value={recoverySelectedKind}
+                            on:change={() =>
+                                selectRecoveryKind(recoverySelectedKind)}
+                        >
+                            <option value={null}
+                                >Choose a replaceable kind...</option
+                            >
+                            {#each replaceableKinds as kind}
+                                <option value={kind.value}>{kind.label}</option>
+                            {/each}
+                        </select>
+                    </div>
+
+                    <div class="custom-kind-input">
+                        <label for="custom-kind"
+                            >Or enter custom kind number:</label
+                        >
+                        <input
+                            id="custom-kind"
+                            type="number"
+                            bind:value={recoveryCustomKind}
+                            on:input={handleCustomKindInput}
+                            placeholder="e.g., 10001"
+                            min="0"
+                        />
+                    </div>
+                </div>
+
+                {#if recoverySelectedKind || recoveryCustomKind}
+                    <div class="recovery-results">
+                        {#if isLoadingRecovery}
+                            <div class="loading">Loading events...</div>
+                        {:else if recoveryEvents.length === 0}
+                            <div class="no-events">
+                                No events found for this kind
+                            </div>
+                        {:else}
+                            <div class="events-list">
+                                {#each recoveryEvents as event}
+                                    {@const isCurrent = isCurrentVersion(event)}
+                                    <div
+                                        class="event-item"
+                                        class:old-version={!isCurrent}
+                                    >
+                                        <div class="event-header">
+                                            <span class="event-kind"
+                                                >Kind {event.kind}</span
+                                            >
+                                            <span class="event-timestamp">
+                                                {new Date(
+                                                    event.created_at * 1000,
+                                                ).toLocaleString()}
+                                            </span>
+                                            {#if !isCurrent}
+                                                <button
+                                                    class="repost-button"
+                                                    on:click={() =>
+                                                        repostEvent(event)}
+                                                >
+                                                    🔄 Repost
+                                                </button>
+                                            {/if}
+                                        </div>
+
+                                        <div class="event-content">
+                                            {#if event.kind === 0}
+                                                <!-- Profile metadata -->
+                                                {#if event.content}
+                                                    <div
+                                                        class="profile-content"
+                                                    >
+                                                        <pre>{JSON.stringify(
+                                                                JSON.parse(
+                                                                    event.content,
+                                                                ),
+                                                                null,
+                                                                2,
+                                                            )}</pre>
+                                                    </div>
+                                                {/if}
+                                            {:else if event.kind === 3}
+                                                <!-- Follow list -->
+                                                <div class="follow-list">
+                                                    <p>
+                                                        Follows {event.tags.filter(
+                                                            (t) => t[0] === "p",
+                                                        ).length} people
+                                                    </p>
+                                                    <div class="follow-tags">
+                                                        {#each event.tags.filter((t) => t[0] === "p") as tag}
+                                                            <span
+                                                                class="pubkey-tag"
+                                                                >{tag[1].substring(
+                                                                    0,
+                                                                    8,
+                                                                )}...</span
+                                                            >
+                                                        {/each}
+                                                    </div>
+                                                </div>
+                                            {:else}
+                                                <!-- Generic content -->
+                                                <div class="generic-content">
+                                                    <pre>{event.content}</pre>
+                                                </div>
+                                            {/if}
+                                        </div>
+
+                                        <div class="event-tags">
+                                            {#each event.tags as tag}
+                                                <span class="tag"
+                                                    >{tag[0]}: {tag[1]}</span
+                                                >
+                                            {/each}
+                                        </div>
+                                    </div>
+                                {/each}
+                            </div>
+
+                            {#if recoveryHasMore}
+                                <button
+                                    class="load-more"
+                                    on:click={loadRecoveryEvents}
+                                    disabled={isLoadingRecovery}
+                                >
+                                    Load More Events
+                                </button>
+                            {/if}
+                        {/if}
+                    </div>
+                {/if}
+            </div>
         {:else if searchTabs.some((tab) => tab.id === selectedTab)}
             {#each searchTabs as searchTab}
                 {#if searchTab.id === selectedTab}
@@ -4610,4 +4939,231 @@
             font-size: 0.8rem;
         }
     }
+
+    /* Recovery Tab Styles */
+    .recovery-tab {
+        padding: 20px;
+        max-width: 1200px;
+        margin: 0 auto;
+    }
+
+    .recovery-header {
+        margin-bottom: 30px;
+        text-align: center;
+    }
+
+    .recovery-header h2 {
+        margin: 0 0 10px 0;
+        color: var(--text-color);
+    }
+
+    .recovery-header p {
+        margin: 0;
+        color: var(--text-color);
+        opacity: 0.7;
+    }
+
+    .recovery-controls {
+        display: flex;
+        gap: 20px;
+        align-items: center;
+        margin-bottom: 30px;
+        padding: 20px;
+        background: var(--bg-color);
+        border-radius: 8px;
+        flex-wrap: wrap;
+    }
+
+    .kind-selector {
+        display: flex;
+        flex-direction: column;
+        gap: 5px;
+    }
+
+    .kind-selector label {
+        font-weight: 500;
+        color: var(--text-color);
+    }
+
+    .kind-selector select {
+        padding: 8px 12px;
+        border: 1px solid var(--border-color);
+        border-radius: 4px;
+        background: var(--bg-color);
+        color: var(--text-color);
+        min-width: 300px;
+    }
+
+    .custom-kind-input {
+        display: flex;
+        flex-direction: column;
+        gap: 5px;
+    }
+
+    .custom-kind-input label {
+        font-weight: 500;
+        color: var(--text-color);
+    }
+
+    .custom-kind-input input {
+        padding: 8px 12px;
+        border: 1px solid var(--border-color);
+        border-radius: 4px;
+        background: var(--bg-color);
+        color: var(--text-color);
+        min-width: 200px;
+    }
+
+    .custom-kind-input input::placeholder {
+        color: var(--text-color);
+        opacity: 0.6;
+    }
+
+    .recovery-results {
+        margin-top: 20px;
+    }
+
+    .loading,
+    .no-events {
+        text-align: center;
+        padding: 40px;
+        color: var(--text-color);
+        opacity: 0.7;
+    }
+
+    .events-list {
+        display: flex;
+        flex-direction: column;
+        gap: 15px;
+    }
+
+    .event-item {
+        background: var(--bg-color);
+        border: 1px solid var(--border-color);
+        border-radius: 8px;
+        padding: 20px;
+        transition: all 0.2s ease;
+    }
+
+    .event-item.old-version {
+        opacity: 0.7;
+        border-color: #ffc107;
+        background: rgba(255, 193, 7, 0.1);
+    }
+
+    .event-header {
+        display: flex;
+        justify-content: space-between;
+        align-items: center;
+        margin-bottom: 15px;
+        flex-wrap: wrap;
+        gap: 10px;
+    }
+
+    .event-kind {
+        font-weight: 600;
+        color: var(--primary);
+    }
+
+    .event-timestamp {
+        color: var(--text-color);
+        font-size: 0.9em;
+        opacity: 0.7;
+    }
+
+    .repost-button {
+        background: var(--primary);
+        color: white;
+        border: none;
+        padding: 6px 12px;
+        border-radius: 4px;
+        cursor: pointer;
+        font-size: 0.9em;
+        transition: background 0.2s ease;
+    }
+
+    .repost-button:hover {
+        background: #00acc1;
+    }
+
+    .event-content {
+        margin-bottom: 15px;
+    }
+
+    .profile-content pre,
+    .generic-content pre {
+        background: var(--bg-color);
+        padding: 15px;
+        border-radius: 4px;
+        overflow-x: auto;
+        font-size: 0.9em;
+        margin: 0;
+        border: 1px solid var(--border-color);
+    }
+
+    .follow-list p {
+        margin: 0 0 10px 0;
+        font-weight: 500;
+        color: var(--text-color);
+    }
+
+    .follow-tags {
+        display: flex;
+        flex-wrap: wrap;
+        gap: 8px;
+    }
+
+    .pubkey-tag {
+        background: var(--bg-color);
+        padding: 4px 8px;
+        border-radius: 4px;
+        font-family: monospace;
+        font-size: 0.8em;
+        border: 1px solid var(--border-color);
+        color: var(--text-color);
+    }
+
+    .event-tags {
+        display: flex;
+        flex-wrap: wrap;
+        gap: 8px;
+    }
+
+    .tag {
+        background: var(--bg-color);
+        padding: 4px 8px;
+        border-radius: 4px;
+        font-size: 0.8em;
+        color: var(--text-color);
+        opacity: 0.7;
+        border: 1px solid var(--border-color);
+    }
+
+    .load-more {
+        width: 100%;
+        padding: 12px;
+        background: var(--primary);
+        color: white;
+        border: none;
+        border-radius: 4px;
+        cursor: pointer;
+        font-size: 1em;
+        margin-top: 20px;
+        transition: background 0.2s ease;
+    }
+
+    .load-more:hover:not(:disabled) {
+        background: #00acc1;
+    }
+
+    .load-more:disabled {
+        opacity: 0.6;
+        cursor: not-allowed;
+    }
+
+    /* Dark theme adjustments for recovery tab */
+    :global(body.dark-theme) .event-item.old-version {
+        background: rgba(255, 193, 7, 0.1);
+        border-color: #ffc107;
+    }
 </style>

app/web/src/nostr.js

@@ -308,7 +308,9 @@ export async function fetchAllEvents(options = {}) {
     limit = 100,
     since = null,
     until = null,
-    authors = null
+    authors = null,
+    kinds = null,
+    tags = null
   } = options;
 
   const filters = {};
@@ -316,6 +318,8 @@ export async function fetchAllEvents(options = {}) {
   if (since) filters.since = since;
   if (until) filters.until = until;
   if (authors) filters.authors = authors;
+  if (kinds) filters.kinds = kinds;
+  if (tags) filters.tags = tags;
   
   // Don't specify kinds filter - this will include all events including delete events (kind 5)
   

pkg/database/get-indexes-from-filter.go

@@ -10,6 +10,7 @@ import (
 	"next.orly.dev/pkg/database/indexes"
 	types2 "next.orly.dev/pkg/database/indexes/types"
 	"next.orly.dev/pkg/encoders/filter"
+	"next.orly.dev/pkg/encoders/tag"
 )
 
 type Range struct {
@@ -145,21 +146,28 @@ func GetIndexesFromFilter(f *filter.F) (idxs []Range, err error) {
 		caEnd.Set(uint64(math.MaxInt64))
 	}
 
+	// Filter out special tags that shouldn't affect index selection
+	var filteredTags *tag.S
 	if f.Tags != nil && f.Tags.Len() > 0 {
-		// sort the tags so they are in iteration order (reverse)
+		filteredTags = tag.NewSWithCap(f.Tags.Len())
-		tmp := *f.Tags
+		for _, t := range *f.Tags {
-		sort.Slice(
+			// Skip the special "show_all_versions" tag
-			tmp, func(i, j int) bool {
+			if bytes.Equal(t.Key(), []byte("show_all_versions")) {
-				return bytes.Compare(tmp[i].Key(), tmp[j].Key()) > 0
+				continue
-			},
+			}
-		)
+			filteredTags.Append(t)
+		}
+		// sort the filtered tags so they are in iteration order (reverse)
+		if filteredTags.Len() > 0 {
+			sort.Sort(filteredTags)
+		}
 	}
 
 	// TagKindPubkey tkp
-	if f.Kinds != nil && f.Kinds.Len() > 0 && f.Authors != nil && f.Authors.Len() > 0 && f.Tags != nil && f.Tags.Len() > 0 {
+	if f.Kinds != nil && f.Kinds.Len() > 0 && f.Authors != nil && f.Authors.Len() > 0 && filteredTags != nil && filteredTags.Len() > 0 {
 		for _, k := range f.Kinds.ToUint16() {
 			for _, author := range f.Authors.T {
-				for _, t := range *f.Tags {
+				for _, t := range *filteredTags {
 					// accept single-letter keys like "e" or filter-style keys like "#e"
 					if t.Len() >= 2 && (len(t.Key()) == 1 || (len(t.Key()) == 2 && t.Key()[0] == '#')) {
 						kind := new(types2.Uint16)
@@ -206,9 +214,9 @@ func GetIndexesFromFilter(f *filter.F) (idxs []Range, err error) {
 	}
 
 	// TagKind tkc
-	if f.Kinds != nil && f.Kinds.Len() > 0 && f.Tags != nil && f.Tags.Len() > 0 {
+	if f.Kinds != nil && f.Kinds.Len() > 0 && filteredTags != nil && filteredTags.Len() > 0 {
 		for _, k := range f.Kinds.ToUint16() {
-			for _, t := range *f.Tags {
+			for _, t := range *filteredTags {
 				if t.Len() >= 2 && (len(t.Key()) == 1 || (len(t.Key()) == 2 && t.Key()[0] == '#')) {
 					kind := new(types2.Uint16)
 					kind.Set(k)
@@ -249,9 +257,9 @@ func GetIndexesFromFilter(f *filter.F) (idxs []Range, err error) {
 	}
 
 	// TagPubkey tpc
-	if f.Authors != nil && f.Authors.Len() > 0 && f.Tags != nil && f.Tags.Len() > 0 {
+	if f.Authors != nil && f.Authors.Len() > 0 && filteredTags != nil && filteredTags.Len() > 0 {
 		for _, author := range f.Authors.T {
-			for _, t := range *f.Tags {
+			for _, t := range *filteredTags {
 				if t.Len() >= 2 && (len(t.Key()) == 1 || (len(t.Key()) == 2 && t.Key()[0] == '#')) {
 					var p *types2.PubHash
 					log.I.S(author)
@@ -293,8 +301,8 @@ func GetIndexesFromFilter(f *filter.F) (idxs []Range, err error) {
 	}
 
 	// Tag tc-
-	if f.Tags != nil && f.Tags.Len() > 0 && (f.Authors == nil || f.Authors.Len() == 0) && (f.Kinds == nil || f.Kinds.Len() == 0) {
+	if filteredTags != nil && filteredTags.Len() > 0 && (f.Authors == nil || f.Authors.Len() == 0) && (f.Kinds == nil || f.Kinds.Len() == 0) {
-		for _, t := range *f.Tags {
+		for _, t := range *filteredTags {
 			if t.Len() >= 2 && (len(t.Key()) == 1 || (len(t.Key()) == 2 && t.Key()[0] == '#')) {
 				keyBytes := t.Key()
 				key := new(types2.Letter)
@@ -353,7 +361,7 @@ func GetIndexesFromFilter(f *filter.F) (idxs []Range, err error) {
 	}
 
 	// Kind kc-
-	if f.Kinds != nil && f.Kinds.Len() > 0 && (f.Authors == nil || f.Authors.Len() == 0) && (f.Tags == nil || f.Tags.Len() == 0) {
+	if f.Kinds != nil && f.Kinds.Len() > 0 && (f.Authors == nil || f.Authors.Len() == 0) && (filteredTags == nil || filteredTags.Len() == 0) {
 		for _, k := range f.Kinds.ToUint16() {
 			kind := new(types2.Uint16)
 			kind.Set(k)

pkg/database/query-events.go

@@ -38,10 +38,17 @@ func CheckExpiration(ev *event.E) (expired bool) {
 func (d *D) QueryEvents(c context.Context, f *filter.F) (
 	evs event.S, err error,
 ) {
-	return d.QueryEventsWithOptions(c, f, true)
+	return d.QueryEventsWithOptions(c, f, true, false)
 }
 
-func (d *D) QueryEventsWithOptions(c context.Context, f *filter.F, includeDeleteEvents bool) (
+// QueryAllVersions queries events and returns all versions of replaceable events
+func (d *D) QueryAllVersions(c context.Context, f *filter.F) (
+	evs event.S, err error,
+) {
+	return d.QueryEventsWithOptions(c, f, true, true)
+}
+
+func (d *D) QueryEventsWithOptions(c context.Context, f *filter.F, includeDeleteEvents bool, showAllVersions bool) (
 	evs event.S, err error,
 ) {
 	// if there is Ids in the query, this overrides anything else
@@ -428,6 +435,9 @@ func (d *D) QueryEventsWithOptions(c context.Context, f *filter.F, includeDelete
 				if deletionsByKindPubkey[key] && !isIdInFilter {
 					// This replaceable event has been deleted, skip it
 					continue
+				} else if showAllVersions {
+					// If showAllVersions is true, treat replaceable events as regular events
+					regularEvents = append(regularEvents, ev)
 				} else {
 					// Normal replaceable event handling
 					existing, exists := replaceableEvents[key]
@@ -459,20 +469,25 @@ func (d *D) QueryEventsWithOptions(c context.Context, f *filter.F, includeDelete
 					}
 				}
 
-				// Initialize the inner map if it doesn't exist
+				if showAllVersions {
-				if _, exists := paramReplaceableEvents[key]; !exists {
+					// If showAllVersions is true, treat parameterized replaceable events as regular events
-					paramReplaceableEvents[key] = make(map[string]*event.E)
+					regularEvents = append(regularEvents, ev)
-				}
+				} else {
+					// Initialize the inner map if it doesn't exist
+					if _, exists := paramReplaceableEvents[key]; !exists {
+						paramReplaceableEvents[key] = make(map[string]*event.E)
+					}
 
-				// Check if we already have an event with this 'd' tag value
+					// Check if we already have an event with this 'd' tag value
-				existing, exists := paramReplaceableEvents[key][dValue]
+					existing, exists := paramReplaceableEvents[key][dValue]
-				// Only keep the newer event, regardless of processing order
+					// Only keep the newer event, regardless of processing order
-				if !exists {
+					if !exists {
-					// No existing event, add this one
+						// No existing event, add this one
-					paramReplaceableEvents[key][dValue] = ev
+						paramReplaceableEvents[key][dValue] = ev
-				} else if ev.CreatedAt > existing.CreatedAt {
+					} else if ev.CreatedAt > existing.CreatedAt {
-					// This event is newer than the existing one, replace it
+						// This event is newer than the existing one, replace it
-					paramReplaceableEvents[key][dValue] = ev
+						paramReplaceableEvents[key][dValue] = ev
+					}
 				}
 				// If this event is older than the existing one, ignore it
 			} else {
@@ -528,7 +543,7 @@ func (d *D) QueryDeleteEventsByTargetId(c context.Context, targetEventId []byte)
 	}
 
 	// Query for the delete events
-	if evs, err = d.QueryEventsWithOptions(c, f, true); chk.E(err) {
+	if evs, err = d.QueryEventsWithOptions(c, f, true, false); chk.E(err) {
 		return
 	}
 

pkg/version/version

@@ -1 +1 @@
-v0.17.3
+v0.17.4