From 4edd4e7a44e9e26bdda0e493a8448b235a58e935 Mon Sep 17 00:00:00 2001 From: woikos Date: Wed, 28 Jan 2026 10:23:28 +0100 Subject: [PATCH] Fix compact format detection for legacy events with 0x01 IDs Legacy events stored before compact format (v6 migration) may have IDs that start with byte 0x01. The v6 migration incorrectly skipped these events because it checked `eventData[0] == CompactFormatVersion` before attempting to decode, causing them to never receive SerialEventId (sei) mappings. This fix: 1. Changes compact format detection to try sei lookup first - if the mapping doesn't exist, fall back to legacy binary format instead of returning an error 2. Adds v9 migration (BackfillMissingSerialEventIdMappings) that finds legacy events with IDs starting with 0x01 that were skipped by the v6 migration and creates their sei mappings 3. Quiets the "Key not found" logging for sei lookups since missing mappings are now expected and handled gracefully Files modified: - pkg/database/fetch-events-by-serials.go: Fallback to legacy format - pkg/database/fetch-event-by-serial.go: Fallback to legacy format - pkg/database/export.go: Fallback to legacy format - pkg/database/serial_cache.go: Quiet sei lookup logging - pkg/database/migrations.go: Add v9 sei backfill migration Co-Authored-By: Claude Opus 4.5 --- pkg/database/export.go | 13 +- pkg/database/fetch-event-by-serial.go | 29 +-- pkg/database/fetch-events-by-serials.go | 54 ++--- pkg/database/migrations.go | 251 +++++++++++++++++++++++- pkg/database/serial_cache.go | 4 +- 5 files changed, 307 insertions(+), 44 deletions(-) diff --git a/pkg/database/export.go b/pkg/database/export.go index e7058de..f77028f 100644 --- a/pkg/database/export.go +++ b/pkg/database/export.go @@ -50,14 +50,17 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) { // Helper function to unmarshal event data (handles both legacy and compact formats) unmarshalEventData := func(val []byte, ser *types.Uint40) (*event.E, error) { // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(val) > 0 && val[0] == CompactFormatVersion { - // Get event ID from SerialEventId table + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Can't decode without event ID - skip - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + return UnmarshalCompactEvent(val, eventId, resolver) } - return UnmarshalCompactEvent(val, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal } // Legacy binary format diff --git a/pkg/database/fetch-event-by-serial.go b/pkg/database/fetch-event-by-serial.go index 92561a9..b3fde40 100644 --- a/pkg/database/fetch-event-by-serial.go +++ b/pkg/database/fetch-event-by-serial.go @@ -41,15 +41,17 @@ func (d *D) FetchEventBySerial(ser *types.Uint40) (ev *event.E, err error) { if len(key) >= dataStart+size { eventData := key[dataStart : dataStart+size] - // Check if this is compact format + // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(eventData) > 0 && eventData[0] == CompactFormatVersion { eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - return nil, fmt.Errorf("compact format inline but no event ID mapping for serial %d: %w", ser.Get(), idErr) + if idErr == nil { + // SerialEventId mapping exists - this is compact format + return UnmarshalCompactEvent(eventData, eventId, resolver) } - return UnmarshalCompactEvent(eventData, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal } // Legacy binary format @@ -106,17 +108,18 @@ func (d *D) FetchEventBySerial(ser *types.Uint40) (ev *event.E, err error) { return } - // Check if this is compact format + // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(v) > 0 && v[0] == CompactFormatVersion { eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - err = fmt.Errorf("compact format evt but no event ID mapping for serial %d: %w", ser.Get(), idErr) + if idErr == nil { + // SerialEventId mapping exists - this is compact format + ev, err = UnmarshalCompactEvent(v, eventId, resolver) return } - ev, err = UnmarshalCompactEvent(v, eventId, resolver) - return + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal } // Check if we have valid data before attempting to unmarshal diff --git a/pkg/database/fetch-events-by-serials.go b/pkg/database/fetch-events-by-serials.go index 8c11575..cc42cdc 100644 --- a/pkg/database/fetch-events-by-serials.go +++ b/pkg/database/fetch-events-by-serials.go @@ -111,17 +111,19 @@ func (d *D) fetchSmallEventWithIterator(txn *badger.Txn, ser *types.Uint40, it * eventData := key[dataStart : dataStart+size] // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(eventData) > 0 && eventData[0] == CompactFormatVersion { - // This is compact format stored in sev - need to decode with resolver - resolver := NewDatabaseSerialResolver(d, d.serialCache) + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - log.W.F("fetchSmallEventWithIterator: compact format but no event ID mapping for serial %d: %v", ser.Get(), idErr) - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + resolver := NewDatabaseSerialResolver(d, d.serialCache) + return UnmarshalCompactEvent(eventData, eventId, resolver) } - return UnmarshalCompactEvent(eventData, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal + log.T.F("fetchSmallEventWithIterator: no sei mapping for serial %d, trying legacy format", ser.Get()) } // Legacy binary format @@ -207,17 +209,19 @@ func (d *D) fetchSmallEvent(txn *badger.Txn, ser *types.Uint40) (ev *event.E, er eventData := key[dataStart : dataStart+size] // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(eventData) > 0 && eventData[0] == CompactFormatVersion { - // This is compact format stored in sev - need to decode with resolver - resolver := NewDatabaseSerialResolver(d, d.serialCache) + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - log.W.F("fetchSmallEvent: compact format but no event ID mapping for serial %d: %v", ser.Get(), idErr) - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + resolver := NewDatabaseSerialResolver(d, d.serialCache) + return UnmarshalCompactEvent(eventData, eventId, resolver) } - return UnmarshalCompactEvent(eventData, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal + log.T.F("fetchSmallEvent: no sei mapping for serial %d, trying legacy format", ser.Get()) } // Legacy binary format @@ -252,17 +256,19 @@ func (d *D) fetchLegacyEvent(txn *badger.Txn, ser *types.Uint40) (ev *event.E, e } // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(v) > 0 && v[0] == CompactFormatVersion { - // This is compact format stored in evt - need to decode with resolver - resolver := NewDatabaseSerialResolver(d, d.serialCache) + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - log.W.F("fetchLegacyEvent: compact format but no event ID mapping for serial %d: %v", ser.Get(), idErr) - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + resolver := NewDatabaseSerialResolver(d, d.serialCache) + return UnmarshalCompactEvent(v, eventId, resolver) } - return UnmarshalCompactEvent(v, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal + log.T.F("fetchLegacyEvent: no sei mapping for serial %d, trying legacy format", ser.Get()) } // Legacy binary format diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go index daad688..cee22a1 100644 --- a/pkg/database/migrations.go +++ b/pkg/database/migrations.go @@ -19,7 +19,7 @@ import ( ) const ( - currentVersion uint32 = 8 + currentVersion uint32 = 9 ) func (d *D) RunMigrations() { @@ -124,6 +124,15 @@ func (d *D) RunMigrations() { // bump to version 8 _ = d.writeVersionTag(8) } + if dbVersion < 9 { + log.I.F("migrating to version 9...") + // Backfill SerialEventId mappings for legacy events that were skipped + // during the v6 compact format migration because their ID starts with 0x01 + // (which was mistakenly interpreted as CompactFormatVersion) + d.BackfillMissingSerialEventIdMappings() + // bump to version 9 + _ = d.writeVersionTag(9) + } } // writeVersionTag writes a new version tag key to the database (no value) @@ -1268,3 +1277,243 @@ func (d *D) BackfillETagGraph() { log.I.F("e-tag graph backfill complete: created %d bidirectional edges", createdEdges) } + +// BackfillMissingSerialEventIdMappings finds legacy events that were incorrectly +// skipped during the v6 compact format migration and creates their SerialEventId +// mappings. This fixes events whose ID happens to start with byte 0x01, which was +// mistakenly interpreted as CompactFormatVersion during the original migration. +// +// The v6 migration had this check: +// +// if len(eventData) > 0 && eventData[0] == CompactFormatVersion { continue } +// +// This caused legacy events with IDs starting with 0x01 to be skipped, leaving +// them without sei mappings and causing "Key not found" errors when fetching. +func (d *D) BackfillMissingSerialEventIdMappings() { + log.I.F("backfilling missing SerialEventId mappings for legacy events...") + var err error + + type LegacyEvent struct { + Serial uint64 + EventData []byte + IsInline bool // true if from sev, false if from evt + } + + var legacyEvents []LegacyEvent + var alreadyHasMapping int + var skippedCompact int + + // First pass: find legacy events that don't have sei mappings + if err = d.View(func(txn *badger.Txn) error { + // Process evt (large events) table + evtPrf := new(bytes.Buffer) + if err = indexes.EventEnc(nil).MarshalWrite(evtPrf); chk.E(err) { + return err + } + it := txn.NewIterator(badger.IteratorOptions{Prefix: evtPrf.Bytes()}) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + key := item.KeyCopy(nil) + + // Extract serial from key + ser := indexes.EventVars() + if err = indexes.EventDec(ser).UnmarshalRead(bytes.NewBuffer(key)); chk.E(err) { + continue + } + + // Check if this serial already has an sei mapping + seiKey := new(bytes.Buffer) + if err = indexes.SerialEventIdEnc(ser).MarshalWrite(seiKey); err == nil { + if _, getErr := txn.Get(seiKey.Bytes()); getErr == nil { + // Already has mapping + alreadyHasMapping++ + continue + } + } + + var val []byte + if val, err = item.ValueCopy(nil); chk.E(err) { + continue + } + + // Only process if first byte is 0x01 (these were the ones skipped) + // Events with other first bytes would have been migrated correctly + if len(val) == 0 || val[0] != CompactFormatVersion { + continue + } + + // Verify this is actually legacy format by checking size + // Legacy format: 32 (ID) + 32 (Pubkey) + varints + 64 (Sig) = ~135+ bytes minimum + // Compact format is smaller, typically < 100 bytes for simple events + if len(val) < 130 { + // Likely actually compact format, skip + skippedCompact++ + continue + } + + legacyEvents = append(legacyEvents, LegacyEvent{ + Serial: ser.Get(), + EventData: val, + IsInline: false, + }) + } + it.Close() + + // Process sev (small inline events) table + sevPrf := new(bytes.Buffer) + if err = indexes.SmallEventEnc(nil).MarshalWrite(sevPrf); chk.E(err) { + return err + } + it2 := txn.NewIterator(badger.IteratorOptions{Prefix: sevPrf.Bytes()}) + defer it2.Close() + + for it2.Rewind(); it2.Valid(); it2.Next() { + item := it2.Item() + key := item.KeyCopy(nil) + + // Extract serial and data from inline key + if len(key) <= 8+2 { + continue + } + + // Extract serial + ser := new(types.Uint40) + if err = ser.UnmarshalRead(bytes.NewReader(key[3:8])); chk.E(err) { + continue + } + + // Check if this serial already has an sei mapping + seiKey := new(bytes.Buffer) + if err = indexes.SerialEventIdEnc(ser).MarshalWrite(seiKey); err == nil { + if _, getErr := txn.Get(seiKey.Bytes()); getErr == nil { + // Already has mapping + alreadyHasMapping++ + continue + } + } + + // Extract size and data + sizeIdx := 8 + size := int(key[sizeIdx])<<8 | int(key[sizeIdx+1]) + dataStart := sizeIdx + 2 + if len(key) < dataStart+size { + continue + } + eventData := key[dataStart : dataStart+size] + + // Only process if first byte is 0x01 + if len(eventData) == 0 || eventData[0] != CompactFormatVersion { + continue + } + + // Verify this is actually legacy format by checking size + if len(eventData) < 130 { + skippedCompact++ + continue + } + + legacyEvents = append(legacyEvents, LegacyEvent{ + Serial: ser.Get(), + EventData: eventData, + IsInline: true, + }) + } + + return nil + }); chk.E(err) { + log.E.F("failed to scan for legacy events: %v", err) + return + } + + log.I.F("found %d legacy events needing sei mappings (%d already have mappings, %d skipped as compact)", + len(legacyEvents), alreadyHasMapping, skippedCompact) + + if len(legacyEvents) == 0 { + log.I.F("no legacy events need sei mapping backfill") + return + } + + // Create resolver for potential compact conversion + resolver := NewDatabaseSerialResolver(d, d.serialCache) + + // Process each event + var successCount, failCount int + var convertedToCompact int + + for i, le := range legacyEvents { + if err = d.Update(func(txn *badger.Txn) error { + // Decode the legacy event to get the ID + ev := new(event.E) + if err = ev.UnmarshalBinary(bytes.NewBuffer(le.EventData)); err != nil { + log.D.F("backfill: failed to decode event serial %d as legacy format: %v", le.Serial, err) + failCount++ + return nil // Continue with next event + } + + // Verify the event ID actually starts with 0x01 + if len(ev.ID) < 1 || ev.ID[0] != 0x01 { + log.D.F("backfill: event serial %d doesn't have ID starting with 0x01, skipping", le.Serial) + return nil + } + + // Store SerialEventId mapping + if err = d.StoreEventIdSerial(txn, le.Serial, ev.ID); chk.E(err) { + log.W.F("backfill: failed to store sei mapping for serial %d: %v", le.Serial, err) + failCount++ + return nil + } + + // Cache the mapping + d.serialCache.CacheEventId(le.Serial, ev.ID) + + // Also convert to compact format if not already done + ser := new(types.Uint40) + if err = ser.Set(le.Serial); err != nil { + successCount++ + return nil + } + + // Check if cmp entry exists + cmpKey := new(bytes.Buffer) + if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKey); err == nil { + if _, getErr := txn.Get(cmpKey.Bytes()); getErr == nil { + // Already has compact entry + successCount++ + return nil + } + } + + // Create compact format entry + compactData, encErr := MarshalCompactEvent(ev, resolver) + if encErr != nil { + log.D.F("backfill: failed to encode compact event for serial %d: %v", le.Serial, encErr) + successCount++ // sei mapping was successful, just couldn't convert + return nil + } + + if err = txn.Set(cmpKey.Bytes(), compactData); chk.E(err) { + log.D.F("backfill: failed to store compact event for serial %d: %v", le.Serial, err) + successCount++ // sei mapping was successful + return nil + } + + convertedToCompact++ + successCount++ + return nil + }); chk.E(err) { + log.W.F("backfill: transaction failed for serial %d: %v", le.Serial, err) + failCount++ + continue + } + + // Log progress every 1000 events + if (i+1)%1000 == 0 { + log.I.F("backfill progress: %d/%d events processed", i+1, len(legacyEvents)) + } + } + + log.I.F("SerialEventId backfill complete: %d successful, %d failed, %d also converted to compact format", + successCount, failCount, convertedToCompact) +} diff --git a/pkg/database/serial_cache.go b/pkg/database/serial_cache.go index 6764cf0..224d24b 100644 --- a/pkg/database/serial_cache.go +++ b/pkg/database/serial_cache.go @@ -246,7 +246,9 @@ func (d *D) GetEventIdBySerial(ser *types.Uint40) (eventId []byte, err error) { err = d.View(func(txn *badger.Txn) error { item, gerr := txn.Get(keyBuf.Bytes()) - if chk.E(gerr) { + if gerr != nil { + // Don't log ErrKeyNotFound - it's expected for legacy events + // that don't have SerialEventId mappings return gerr }