import { beforeEach, describe, expect, it, vi } from "vitest"; import type { NDKEvent } from "@nostr-dev-kit/ndk"; import { deduplicateAndCombineEvents, deduplicateContentEvents, getEventCoordinate, isReplaceableEvent, } from "../../src/lib/utils/eventDeduplication"; // Mock NDKEvent for testing class MockNDKEvent { id: string; kind: number; pubkey: string; created_at: number; content: string; tags: string[][]; constructor( id: string, kind: number, pubkey: string, created_at: number, dTag: string, content: string = "", ) { this.id = id; this.kind = kind; this.pubkey = pubkey; this.created_at = created_at; this.content = content; this.tags = [["d", dTag]]; } tagValue(tagName: string): string | undefined { const tag = this.tags.find((t) => t[0] === tagName); return tag ? tag[1] : undefined; } } describe("Relay Deduplication Behavior Tests", () => { let mockEvents: MockNDKEvent[]; beforeEach(() => { // Create test events with different timestamps mockEvents = [ // Older version of a publication content event new MockNDKEvent( "event1", 30041, "pubkey1", 1000, "chapter-1", "Old content", ), // Newer version of the same publication content event new MockNDKEvent( "event2", 30041, "pubkey1", 2000, "chapter-1", "Updated content", ), // Different publication content event new MockNDKEvent( "event3", 30041, "pubkey1", 1500, "chapter-2", "Different content", ), // Publication index event (should not be deduplicated) new MockNDKEvent( "event4", 30040, "pubkey1", 1200, "book-1", "Index content", ), // Regular text note (should not be deduplicated) new MockNDKEvent("event5", 1, "pubkey1", 1300, "", "Regular note"), ]; }); describe("Addressable Event Deduplication", () => { it("should keep only the most recent version of addressable events by coordinate", () => { // Test the deduplication logic for content events const eventSets = [ new Set(mockEvents.filter((e) => e.kind === 30041) as NDKEvent[]), ]; const result = deduplicateContentEvents(eventSets); // Should have 2 unique coordinates: chapter-1 and chapter-2 expect(result.size).toBe(2); // Should keep the newer version of chapter-1 const chapter1Event = result.get("30041:pubkey1:chapter-1"); expect(chapter1Event?.id).toBe("event2"); expect(chapter1Event?.content).toBe("Updated content"); // Should keep chapter-2 const chapter2Event = result.get("30041:pubkey1:chapter-2"); expect(chapter2Event?.id).toBe("event3"); }); it("should handle events with missing d-tags gracefully", () => { const eventWithoutDTag = new MockNDKEvent( "event6", 30041, "pubkey1", 1400, "", "No d-tag", ); eventWithoutDTag.tags = []; // Remove d-tag const eventSets = [new Set([eventWithoutDTag] as NDKEvent[])]; const result = deduplicateContentEvents(eventSets); // Should not include events without d-tags expect(result.size).toBe(0); }); it("should handle events with missing timestamps", () => { const eventWithoutTimestamp = new MockNDKEvent( "event7", 30041, "pubkey1", 0, "chapter-3", "No timestamp", ); const eventWithTimestamp = new MockNDKEvent( "event8", 30041, "pubkey1", 1500, "chapter-3", "With timestamp", ); const eventSets = [ new Set([eventWithoutTimestamp, eventWithTimestamp] as NDKEvent[]), ]; const result = deduplicateContentEvents(eventSets); // Should prefer the event with timestamp const chapter3Event = result.get("30041:pubkey1:chapter-3"); expect(chapter3Event?.id).toBe("event8"); }); }); describe("Mixed Event Type Deduplication", () => { it("should only deduplicate addressable events (kinds 30000-39999)", () => { const result = deduplicateAndCombineEvents( [mockEvents[4]] as NDKEvent[], // Regular text note new Set([mockEvents[3]] as NDKEvent[]), // Publication index new Set([mockEvents[0], mockEvents[1], mockEvents[2]] as NDKEvent[]), // Content events ); // Should have 4 events total: // - 1 regular text note (not deduplicated) // - 1 publication index (not deduplicated) // - 2 unique content events (deduplicated from 3) expect(result.length).toBe(4); // Verify the content events were deduplicated const contentEvents = result.filter((e) => e.kind === 30041); expect(contentEvents.length).toBe(2); // Verify the newer version was kept const newerEvent = contentEvents.find((e) => e.id === "event2"); expect(newerEvent).toBeDefined(); }); it("should handle non-addressable events correctly", () => { const regularEvents = [ new MockNDKEvent("note1", 1, "pubkey1", 1000, "", "Note 1"), new MockNDKEvent("note2", 1, "pubkey1", 2000, "", "Note 2"), new MockNDKEvent("profile1", 0, "pubkey1", 1500, "", "Profile 1"), ]; const result = deduplicateAndCombineEvents( regularEvents as NDKEvent[], new Set(), new Set(), ); // All regular events should be included (no deduplication) expect(result.length).toBe(3); }); }); describe("Coordinate System Validation", () => { it("should correctly identify event coordinates", () => { const event = new MockNDKEvent( "test", 30041, "pubkey123", 1000, "test-chapter", ); const coordinate = getEventCoordinate(event as NDKEvent); expect(coordinate).toBe("30041:pubkey123:test-chapter"); }); it("should handle d-tags with colons correctly", () => { const event = new MockNDKEvent( "test", 30041, "pubkey123", 1000, "chapter:with:colons", ); const coordinate = getEventCoordinate(event as NDKEvent); expect(coordinate).toBe("30041:pubkey123:chapter:with:colons"); }); it("should return null for non-replaceable events", () => { const event = new MockNDKEvent("test", 1, "pubkey123", 1000, ""); const coordinate = getEventCoordinate(event as NDKEvent); expect(coordinate).toBeNull(); }); }); describe("Replaceable Event Detection", () => { it("should correctly identify replaceable events", () => { const addressableEvent = new MockNDKEvent( "test", 30041, "pubkey123", 1000, "test", ); const regularEvent = new MockNDKEvent("test", 1, "pubkey123", 1000, ""); expect(isReplaceableEvent(addressableEvent as NDKEvent)).toBe(true); expect(isReplaceableEvent(regularEvent as NDKEvent)).toBe(false); }); it("should handle edge cases of replaceable event ranges", () => { const event29999 = new MockNDKEvent( "test", 29999, "pubkey123", 1000, "test", ); const event30000 = new MockNDKEvent( "test", 30000, "pubkey123", 1000, "test", ); const event39999 = new MockNDKEvent( "test", 39999, "pubkey123", 1000, "test", ); const event40000 = new MockNDKEvent( "test", 40000, "pubkey123", 1000, "test", ); expect(isReplaceableEvent(event29999 as NDKEvent)).toBe(false); expect(isReplaceableEvent(event30000 as NDKEvent)).toBe(true); expect(isReplaceableEvent(event39999 as NDKEvent)).toBe(true); expect(isReplaceableEvent(event40000 as NDKEvent)).toBe(false); }); }); describe("Edge Cases", () => { it("should handle empty event sets", () => { const result = deduplicateContentEvents([]); expect(result.size).toBe(0); }); it("should handle events with null/undefined values", () => { const invalidEvent = { id: undefined, kind: 30041, pubkey: "pubkey1", created_at: 1000, tagValue: () => undefined, // Return undefined for d-tag } as unknown as NDKEvent; const eventSets = [new Set([invalidEvent])]; const result = deduplicateContentEvents(eventSets); // Should handle gracefully without crashing expect(result.size).toBe(0); }); it("should handle events from different authors with same d-tag", () => { const event1 = new MockNDKEvent( "event1", 30041, "pubkey1", 1000, "same-chapter", "Author 1", ); const event2 = new MockNDKEvent( "event2", 30041, "pubkey2", 1000, "same-chapter", "Author 2", ); const eventSets = [new Set([event1, event2] as NDKEvent[])]; const result = deduplicateContentEvents(eventSets); // Should have 2 events (different coordinates due to different authors) expect(result.size).toBe(2); expect(result.has("30041:pubkey1:same-chapter")).toBe(true); expect(result.has("30041:pubkey2:same-chapter")).toBe(true); }); }); }); describe("Relay Behavior Simulation", () => { it("should simulate what happens when relays return duplicate events", () => { // Simulate a relay that returns multiple versions of the same event const relayEvents = [ new MockNDKEvent( "event1", 30041, "pubkey1", 1000, "chapter-1", "Old version", ), new MockNDKEvent( "event2", 30041, "pubkey1", 2000, "chapter-1", "New version", ), new MockNDKEvent( "event3", 30041, "pubkey1", 1500, "chapter-1", "Middle version", ), ]; // This simulates what a "bad" relay might return const eventSets = [new Set(relayEvents as NDKEvent[])]; const result = deduplicateContentEvents(eventSets); // Should only keep the newest version expect(result.size).toBe(1); const keptEvent = result.get("30041:pubkey1:chapter-1"); expect(keptEvent?.id).toBe("event2"); expect(keptEvent?.content).toBe("New version"); }); it("should simulate multiple relays returning different versions", () => { // Simulate multiple relays returning different versions const relay1Events = [ new MockNDKEvent( "event1", 30041, "pubkey1", 1000, "chapter-1", "Relay 1 version", ), ]; const relay2Events = [ new MockNDKEvent( "event2", 30041, "pubkey1", 2000, "chapter-1", "Relay 2 version", ), ]; const eventSets = [ new Set(relay1Events as NDKEvent[]), new Set(relay2Events as NDKEvent[]), ]; const result = deduplicateContentEvents(eventSets); // Should keep the newest version from any relay expect(result.size).toBe(1); const keptEvent = result.get("30041:pubkey1:chapter-1"); expect(keptEvent?.id).toBe("event2"); expect(keptEvent?.content).toBe("Relay 2 version"); }); }); describe("Real Relay Deduplication Tests", () => { // These tests actually query real relays to see if they deduplicate // Note: These are integration tests and may be flaky due to network conditions it( "should detect if relays are returning duplicate replaceable events", async () => { // This test queries real relays to see if they return duplicates // We'll use a known author who has published multiple versions of content // Known author with multiple publication content events const testAuthor = "npub1z4m7gkva6yxgvdyclc7zp0qt69x9zgn8lu8sllg06wx6432h77qs0k97ks"; // Query for publication content events (kind 30041) from this author // We expect relays to return only the most recent version of each d-tag // This is a placeholder - in a real test, we would: // 1. Query multiple relays for the same author's 30041 events // 2. Check if any relay returns multiple events with the same d-tag // 3. Verify that if duplicates exist, our deduplication logic handles them console.log( "Note: This test would require actual relay queries to verify deduplication behavior", ); console.log("To run this test properly, we would need to:"); console.log("1. Query real relays for replaceable events"); console.log("2. Check if relays return duplicates"); console.log("3. Verify our deduplication logic works on real data"); // For now, we'll just assert that our logic is ready to handle real data expect(true).toBe(true); }, 30000, ); // 30 second timeout for network requests it( "should verify that our deduplication logic works on real relay data", async () => { // This test would: // 1. Fetch real events from relays // 2. Apply our deduplication logic // 3. Verify that the results are correct console.log("Note: This test would require actual relay queries"); console.log("To implement this test, we would need to:"); console.log("1. Set up NDK with real relays"); console.log("2. Fetch events for a known author with multiple versions"); console.log("3. Apply deduplication and verify results"); expect(true).toBe(true); }, 30000, ); }); describe("Practical Relay Behavior Analysis", () => { it("should document what we know about relay deduplication behavior", () => { // This test documents our current understanding of relay behavior // based on the code analysis and the comment from onedev console.log("\n=== RELAY DEDUPLICATION BEHAVIOR ANALYSIS ==="); console.log("\nBased on the code analysis and the comment from onedev:"); console.log("\n1. THEORETICAL BEHAVIOR:"); console.log( " - Relays SHOULD handle deduplication for replaceable events", ); console.log( " - Only the most recent version of each coordinate should be stored", ); console.log( " - Client-side deduplication should only be needed for cached/local events", ); console.log("\n2. REALITY CHECK:"); console.log(" - Not all relays implement deduplication correctly"); console.log( " - Some relays may return multiple versions of the same event", ); console.log( " - Network conditions and relay availability can cause inconsistencies", ); console.log("\n3. ALEXANDRIA'S APPROACH:"); console.log(" - Implements client-side deduplication as a safety net"); console.log( " - Uses coordinate system (kind:pubkey:d-tag) for addressable events", ); console.log( " - Keeps the most recent version based on created_at timestamp", ); console.log(" - Only applies to replaceable events (kinds 30000-39999)"); console.log("\n4. WHY KEEP THE DEDUPLICATION:"); console.log( " - Defensive programming against imperfect relay implementations", ); console.log(" - Handles multiple relay sources with different data"); console.log(" - Works with cached events that might be outdated"); console.log( " - Ensures consistent user experience regardless of relay behavior", ); console.log("\n5. TESTING STRATEGY:"); console.log( " - Unit tests verify our deduplication logic works correctly", ); console.log( " - Integration tests would verify relay behavior (when network allows)", ); console.log( " - Monitoring can help determine if relays improve over time", ); // This test documents our understanding rather than asserting specific behavior expect(true).toBe(true); }); it("should provide recommendations for when to remove deduplication", () => { console.log("\n=== RECOMMENDATIONS FOR REMOVING DEDUPLICATION ==="); console.log("\nThe deduplication logic should be kept until:"); console.log("\n1. RELAY STANDARDS:"); console.log( " - NIP-33 (replaceable events) is widely implemented by relays", ); console.log(" - Relays consistently return only the most recent version"); console.log(" - No major relay implementations return duplicates"); console.log("\n2. TESTING EVIDENCE:"); console.log(" - Real-world testing shows relays don't return duplicates"); console.log(" - Multiple relay operators confirm deduplication behavior"); console.log(" - No user reports of duplicate content issues"); console.log("\n3. MONITORING:"); console.log( " - Add logging to track when deduplication is actually used", ); console.log(" - Monitor relay behavior over time"); console.log(" - Collect metrics on duplicate events found"); console.log("\n4. GRADUAL REMOVAL:"); console.log(" - Make deduplication configurable (on/off)"); console.log( " - Test with deduplication disabled in controlled environments", ); console.log(" - Monitor for issues before removing completely"); console.log("\n5. FALLBACK STRATEGY:"); console.log(" - Keep deduplication as a fallback option"); console.log(" - Allow users to enable it if they experience issues"); console.log(" - Maintain the code for potential future use"); expect(true).toBe(true); }); }); describe("Logging and Monitoring Tests", () => { it("should verify that logging works when duplicates are found", () => { // Mock console.log to capture output const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}); // Create events with duplicates const duplicateEvents = [ new MockNDKEvent( "event1", 30041, "pubkey1", 1000, "chapter-1", "Old version", ), new MockNDKEvent( "event2", 30041, "pubkey1", 2000, "chapter-1", "New version", ), new MockNDKEvent( "event3", 30041, "pubkey1", 1500, "chapter-1", "Middle version", ), ]; const eventSets = [new Set(duplicateEvents as NDKEvent[])]; const result = deduplicateContentEvents(eventSets); // Verify the deduplication worked expect(result.size).toBe(1); // Verify that logging was called expect(consoleSpy).toHaveBeenCalledWith( expect.stringContaining( "[eventDeduplication] Found 2 duplicate events out of 3 total events", ), ); expect(consoleSpy).toHaveBeenCalledWith( expect.stringContaining( "[eventDeduplication] Reduced to 1 unique coordinates", ), ); // Restore console.log consoleSpy.mockRestore(); }); it("should verify that logging works when no duplicates are found", () => { // Mock console.log to capture output const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}); // Create events without duplicates const uniqueEvents = [ new MockNDKEvent( "event1", 30041, "pubkey1", 1000, "chapter-1", "Content 1", ), new MockNDKEvent( "event2", 30041, "pubkey1", 2000, "chapter-2", "Content 2", ), ]; const eventSets = [new Set(uniqueEvents as NDKEvent[])]; const result = deduplicateContentEvents(eventSets); // Verify no deduplication was needed expect(result.size).toBe(2); // Verify that logging was called with "no duplicates" message expect(consoleSpy).toHaveBeenCalledWith( expect.stringContaining( "[eventDeduplication] No duplicates found in 2 events", ), ); // Restore console.log consoleSpy.mockRestore(); }); it("should verify that deduplicateAndCombineEvents logging works", () => { // Mock console.log to capture output const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}); // Create events with duplicates const duplicateEvents = [ new MockNDKEvent( "event1", 30041, "pubkey1", 1000, "chapter-1", "Old version", ), new MockNDKEvent( "event2", 30041, "pubkey1", 2000, "chapter-1", "New version", ), ]; const result = deduplicateAndCombineEvents( [] as NDKEvent[], new Set(), new Set(duplicateEvents as NDKEvent[]), ); // Verify the deduplication worked expect(result.length).toBe(1); // Verify that logging was called expect(consoleSpy).toHaveBeenCalledWith( expect.stringContaining( "[eventDeduplication] deduplicateAndCombineEvents: Found 1 duplicate coordinates", ), ); // Restore console.log consoleSpy.mockRestore(); }); });