diff --git a/pkg/database/export.go b/pkg/database/export.go index e7058de..f77028f 100644 --- a/pkg/database/export.go +++ b/pkg/database/export.go @@ -50,14 +50,17 @@ func (d *D) Export(c context.Context, w io.Writer, pubkeys ...[]byte) { // Helper function to unmarshal event data (handles both legacy and compact formats) unmarshalEventData := func(val []byte, ser *types.Uint40) (*event.E, error) { // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(val) > 0 && val[0] == CompactFormatVersion { - // Get event ID from SerialEventId table + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Can't decode without event ID - skip - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + return UnmarshalCompactEvent(val, eventId, resolver) } - return UnmarshalCompactEvent(val, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal } // Legacy binary format diff --git a/pkg/database/fetch-event-by-serial.go b/pkg/database/fetch-event-by-serial.go index 92561a9..b3fde40 100644 --- a/pkg/database/fetch-event-by-serial.go +++ b/pkg/database/fetch-event-by-serial.go @@ -41,15 +41,17 @@ func (d *D) FetchEventBySerial(ser *types.Uint40) (ev *event.E, err error) { if len(key) >= dataStart+size { eventData := key[dataStart : dataStart+size] - // Check if this is compact format + // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(eventData) > 0 && eventData[0] == CompactFormatVersion { eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - return nil, fmt.Errorf("compact format inline but no event ID mapping for serial %d: %w", ser.Get(), idErr) + if idErr == nil { + // SerialEventId mapping exists - this is compact format + return UnmarshalCompactEvent(eventData, eventId, resolver) } - return UnmarshalCompactEvent(eventData, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal } // Legacy binary format @@ -106,17 +108,18 @@ func (d *D) FetchEventBySerial(ser *types.Uint40) (ev *event.E, err error) { return } - // Check if this is compact format + // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(v) > 0 && v[0] == CompactFormatVersion { eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - err = fmt.Errorf("compact format evt but no event ID mapping for serial %d: %w", ser.Get(), idErr) + if idErr == nil { + // SerialEventId mapping exists - this is compact format + ev, err = UnmarshalCompactEvent(v, eventId, resolver) return } - ev, err = UnmarshalCompactEvent(v, eventId, resolver) - return + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal } // Check if we have valid data before attempting to unmarshal diff --git a/pkg/database/fetch-events-by-serials.go b/pkg/database/fetch-events-by-serials.go index 8c11575..cc42cdc 100644 --- a/pkg/database/fetch-events-by-serials.go +++ b/pkg/database/fetch-events-by-serials.go @@ -111,17 +111,19 @@ func (d *D) fetchSmallEventWithIterator(txn *badger.Txn, ser *types.Uint40, it * eventData := key[dataStart : dataStart+size] // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(eventData) > 0 && eventData[0] == CompactFormatVersion { - // This is compact format stored in sev - need to decode with resolver - resolver := NewDatabaseSerialResolver(d, d.serialCache) + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - log.W.F("fetchSmallEventWithIterator: compact format but no event ID mapping for serial %d: %v", ser.Get(), idErr) - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + resolver := NewDatabaseSerialResolver(d, d.serialCache) + return UnmarshalCompactEvent(eventData, eventId, resolver) } - return UnmarshalCompactEvent(eventData, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal + log.T.F("fetchSmallEventWithIterator: no sei mapping for serial %d, trying legacy format", ser.Get()) } // Legacy binary format @@ -207,17 +209,19 @@ func (d *D) fetchSmallEvent(txn *badger.Txn, ser *types.Uint40) (ev *event.E, er eventData := key[dataStart : dataStart+size] // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(eventData) > 0 && eventData[0] == CompactFormatVersion { - // This is compact format stored in sev - need to decode with resolver - resolver := NewDatabaseSerialResolver(d, d.serialCache) + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - log.W.F("fetchSmallEvent: compact format but no event ID mapping for serial %d: %v", ser.Get(), idErr) - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + resolver := NewDatabaseSerialResolver(d, d.serialCache) + return UnmarshalCompactEvent(eventData, eventId, resolver) } - return UnmarshalCompactEvent(eventData, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal + log.T.F("fetchSmallEvent: no sei mapping for serial %d, trying legacy format", ser.Get()) } // Legacy binary format @@ -252,17 +256,19 @@ func (d *D) fetchLegacyEvent(txn *badger.Txn, ser *types.Uint40) (ev *event.E, e } // Check if this is compact format (starts with version byte 1) + // Note: Legacy events whose ID starts with 0x01 will also match this check, + // so we fall back to legacy format if the SerialEventId mapping doesn't exist. if len(v) > 0 && v[0] == CompactFormatVersion { - // This is compact format stored in evt - need to decode with resolver - resolver := NewDatabaseSerialResolver(d, d.serialCache) + // Try to get event ID mapping - if it exists, this is truly compact format eventId, idErr := d.GetEventIdBySerial(ser) - if idErr != nil { - // Cannot decode compact format without event ID - return error - // DO NOT fall back to legacy unmarshal as compact format is not valid legacy format - log.W.F("fetchLegacyEvent: compact format but no event ID mapping for serial %d: %v", ser.Get(), idErr) - return nil, idErr + if idErr == nil { + // SerialEventId mapping exists - this is compact format + resolver := NewDatabaseSerialResolver(d, d.serialCache) + return UnmarshalCompactEvent(v, eventId, resolver) } - return UnmarshalCompactEvent(v, eventId, resolver) + // No SerialEventId mapping - this is likely a legacy event whose ID starts with 0x01 + // Fall through to legacy unmarshal + log.T.F("fetchLegacyEvent: no sei mapping for serial %d, trying legacy format", ser.Get()) } // Legacy binary format diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go index daad688..cee22a1 100644 --- a/pkg/database/migrations.go +++ b/pkg/database/migrations.go @@ -19,7 +19,7 @@ import ( ) const ( - currentVersion uint32 = 8 + currentVersion uint32 = 9 ) func (d *D) RunMigrations() { @@ -124,6 +124,15 @@ func (d *D) RunMigrations() { // bump to version 8 _ = d.writeVersionTag(8) } + if dbVersion < 9 { + log.I.F("migrating to version 9...") + // Backfill SerialEventId mappings for legacy events that were skipped + // during the v6 compact format migration because their ID starts with 0x01 + // (which was mistakenly interpreted as CompactFormatVersion) + d.BackfillMissingSerialEventIdMappings() + // bump to version 9 + _ = d.writeVersionTag(9) + } } // writeVersionTag writes a new version tag key to the database (no value) @@ -1268,3 +1277,243 @@ func (d *D) BackfillETagGraph() { log.I.F("e-tag graph backfill complete: created %d bidirectional edges", createdEdges) } + +// BackfillMissingSerialEventIdMappings finds legacy events that were incorrectly +// skipped during the v6 compact format migration and creates their SerialEventId +// mappings. This fixes events whose ID happens to start with byte 0x01, which was +// mistakenly interpreted as CompactFormatVersion during the original migration. +// +// The v6 migration had this check: +// +// if len(eventData) > 0 && eventData[0] == CompactFormatVersion { continue } +// +// This caused legacy events with IDs starting with 0x01 to be skipped, leaving +// them without sei mappings and causing "Key not found" errors when fetching. +func (d *D) BackfillMissingSerialEventIdMappings() { + log.I.F("backfilling missing SerialEventId mappings for legacy events...") + var err error + + type LegacyEvent struct { + Serial uint64 + EventData []byte + IsInline bool // true if from sev, false if from evt + } + + var legacyEvents []LegacyEvent + var alreadyHasMapping int + var skippedCompact int + + // First pass: find legacy events that don't have sei mappings + if err = d.View(func(txn *badger.Txn) error { + // Process evt (large events) table + evtPrf := new(bytes.Buffer) + if err = indexes.EventEnc(nil).MarshalWrite(evtPrf); chk.E(err) { + return err + } + it := txn.NewIterator(badger.IteratorOptions{Prefix: evtPrf.Bytes()}) + defer it.Close() + + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + key := item.KeyCopy(nil) + + // Extract serial from key + ser := indexes.EventVars() + if err = indexes.EventDec(ser).UnmarshalRead(bytes.NewBuffer(key)); chk.E(err) { + continue + } + + // Check if this serial already has an sei mapping + seiKey := new(bytes.Buffer) + if err = indexes.SerialEventIdEnc(ser).MarshalWrite(seiKey); err == nil { + if _, getErr := txn.Get(seiKey.Bytes()); getErr == nil { + // Already has mapping + alreadyHasMapping++ + continue + } + } + + var val []byte + if val, err = item.ValueCopy(nil); chk.E(err) { + continue + } + + // Only process if first byte is 0x01 (these were the ones skipped) + // Events with other first bytes would have been migrated correctly + if len(val) == 0 || val[0] != CompactFormatVersion { + continue + } + + // Verify this is actually legacy format by checking size + // Legacy format: 32 (ID) + 32 (Pubkey) + varints + 64 (Sig) = ~135+ bytes minimum + // Compact format is smaller, typically < 100 bytes for simple events + if len(val) < 130 { + // Likely actually compact format, skip + skippedCompact++ + continue + } + + legacyEvents = append(legacyEvents, LegacyEvent{ + Serial: ser.Get(), + EventData: val, + IsInline: false, + }) + } + it.Close() + + // Process sev (small inline events) table + sevPrf := new(bytes.Buffer) + if err = indexes.SmallEventEnc(nil).MarshalWrite(sevPrf); chk.E(err) { + return err + } + it2 := txn.NewIterator(badger.IteratorOptions{Prefix: sevPrf.Bytes()}) + defer it2.Close() + + for it2.Rewind(); it2.Valid(); it2.Next() { + item := it2.Item() + key := item.KeyCopy(nil) + + // Extract serial and data from inline key + if len(key) <= 8+2 { + continue + } + + // Extract serial + ser := new(types.Uint40) + if err = ser.UnmarshalRead(bytes.NewReader(key[3:8])); chk.E(err) { + continue + } + + // Check if this serial already has an sei mapping + seiKey := new(bytes.Buffer) + if err = indexes.SerialEventIdEnc(ser).MarshalWrite(seiKey); err == nil { + if _, getErr := txn.Get(seiKey.Bytes()); getErr == nil { + // Already has mapping + alreadyHasMapping++ + continue + } + } + + // Extract size and data + sizeIdx := 8 + size := int(key[sizeIdx])<<8 | int(key[sizeIdx+1]) + dataStart := sizeIdx + 2 + if len(key) < dataStart+size { + continue + } + eventData := key[dataStart : dataStart+size] + + // Only process if first byte is 0x01 + if len(eventData) == 0 || eventData[0] != CompactFormatVersion { + continue + } + + // Verify this is actually legacy format by checking size + if len(eventData) < 130 { + skippedCompact++ + continue + } + + legacyEvents = append(legacyEvents, LegacyEvent{ + Serial: ser.Get(), + EventData: eventData, + IsInline: true, + }) + } + + return nil + }); chk.E(err) { + log.E.F("failed to scan for legacy events: %v", err) + return + } + + log.I.F("found %d legacy events needing sei mappings (%d already have mappings, %d skipped as compact)", + len(legacyEvents), alreadyHasMapping, skippedCompact) + + if len(legacyEvents) == 0 { + log.I.F("no legacy events need sei mapping backfill") + return + } + + // Create resolver for potential compact conversion + resolver := NewDatabaseSerialResolver(d, d.serialCache) + + // Process each event + var successCount, failCount int + var convertedToCompact int + + for i, le := range legacyEvents { + if err = d.Update(func(txn *badger.Txn) error { + // Decode the legacy event to get the ID + ev := new(event.E) + if err = ev.UnmarshalBinary(bytes.NewBuffer(le.EventData)); err != nil { + log.D.F("backfill: failed to decode event serial %d as legacy format: %v", le.Serial, err) + failCount++ + return nil // Continue with next event + } + + // Verify the event ID actually starts with 0x01 + if len(ev.ID) < 1 || ev.ID[0] != 0x01 { + log.D.F("backfill: event serial %d doesn't have ID starting with 0x01, skipping", le.Serial) + return nil + } + + // Store SerialEventId mapping + if err = d.StoreEventIdSerial(txn, le.Serial, ev.ID); chk.E(err) { + log.W.F("backfill: failed to store sei mapping for serial %d: %v", le.Serial, err) + failCount++ + return nil + } + + // Cache the mapping + d.serialCache.CacheEventId(le.Serial, ev.ID) + + // Also convert to compact format if not already done + ser := new(types.Uint40) + if err = ser.Set(le.Serial); err != nil { + successCount++ + return nil + } + + // Check if cmp entry exists + cmpKey := new(bytes.Buffer) + if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKey); err == nil { + if _, getErr := txn.Get(cmpKey.Bytes()); getErr == nil { + // Already has compact entry + successCount++ + return nil + } + } + + // Create compact format entry + compactData, encErr := MarshalCompactEvent(ev, resolver) + if encErr != nil { + log.D.F("backfill: failed to encode compact event for serial %d: %v", le.Serial, encErr) + successCount++ // sei mapping was successful, just couldn't convert + return nil + } + + if err = txn.Set(cmpKey.Bytes(), compactData); chk.E(err) { + log.D.F("backfill: failed to store compact event for serial %d: %v", le.Serial, err) + successCount++ // sei mapping was successful + return nil + } + + convertedToCompact++ + successCount++ + return nil + }); chk.E(err) { + log.W.F("backfill: transaction failed for serial %d: %v", le.Serial, err) + failCount++ + continue + } + + // Log progress every 1000 events + if (i+1)%1000 == 0 { + log.I.F("backfill progress: %d/%d events processed", i+1, len(legacyEvents)) + } + } + + log.I.F("SerialEventId backfill complete: %d successful, %d failed, %d also converted to compact format", + successCount, failCount, convertedToCompact) +} diff --git a/pkg/database/serial_cache.go b/pkg/database/serial_cache.go index 6764cf0..224d24b 100644 --- a/pkg/database/serial_cache.go +++ b/pkg/database/serial_cache.go @@ -246,7 +246,9 @@ func (d *D) GetEventIdBySerial(ser *types.Uint40) (eventId []byte, err error) { err = d.View(func(txn *badger.Txn) error { item, gerr := txn.Get(keyBuf.Bytes()) - if chk.E(gerr) { + if gerr != nil { + // Don't log ErrKeyNotFound - it's expected for legacy events + // that don't have SerialEventId mappings return gerr }