Browse Source

Optimize ACL follows startup with batched QueryEvents

- Replace manual GetSerialsByRange + FetchEventBySerial loops in
  Configure() and adminRelays() with single batched QueryEvents calls
- Reduces N*(1+M) gRPC round-trips to 1 call per query, cutting ACL
  startup from >2 minutes to sub-second
- Handle Configure errors gracefully instead of os.Exit(1) to support
  empty databases (syncer populates follows from external relays)
- Remove unused database/indexes/types import

Files modified:
- pkg/acl/follows.go: Batched QueryEvents for kind 3 and kind 10002
- cmd/orly-acl-follows/main.go: Graceful error handling on configure
- pkg/version/version: Bump to v0.58.6

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
imwald-v0.58.10
woikos 4 months ago
parent
commit
583ced9b7a
No known key found for this signature in database
  1. 5
      cmd/orly-acl-follows/main.go
  2. 87
      pkg/acl/follows.go
  3. 2
      pkg/version/version

5
cmd/orly-acl-follows/main.go

@ -129,8 +129,9 @@ func main() {
// Create and configure server // Create and configure server
srv := server.New(db, serverCfg, ownsDB) srv := server.New(db, serverCfg, ownsDB)
if err := srv.ConfigureACL(ctx); chk.E(err) { if err := srv.ConfigureACL(ctx); chk.E(err) {
log.E.F("failed to configure ACL: %v", err) // Don't exit on configure error - the syncer will populate follows from
os.Exit(1) // external relays. This handles empty databases gracefully.
log.W.F("ACL configure returned error (will start with 0 follows): %v", err)
} }
// Start server // Start server

87
pkg/acl/follows.go

@ -15,7 +15,6 @@ import (
"lol.mleku.dev/log" "lol.mleku.dev/log"
"next.orly.dev/app/config" "next.orly.dev/app/config"
"next.orly.dev/pkg/database" "next.orly.dev/pkg/database"
"next.orly.dev/pkg/database/indexes/types"
"git.mleku.dev/mleku/nostr/encoders/bech32encoding" "git.mleku.dev/mleku/nostr/encoders/bech32encoding"
"git.mleku.dev/mleku/nostr/encoders/envelopes" "git.mleku.dev/mleku/nostr/encoders/envelopes"
"git.mleku.dev/mleku/nostr/encoders/envelopes/eoseenvelope" "git.mleku.dev/mleku/nostr/encoders/envelopes/eoseenvelope"
@ -104,7 +103,8 @@ func (f *Follows) Configure(cfg ...any) (err error) {
newOwnersSet[hex.EncodeToString(own)] = struct{}{} newOwnersSet[hex.EncodeToString(own)] = struct{}{}
} }
// find admin follow lists (database I/O happens here, but no lock held) // parse admin pubkeys
var adminBinaries [][]byte
for _, admin := range f.cfg.Admins { for _, admin := range f.cfg.Admins {
var adm []byte var adm []byte
if a, e := bech32encoding.NpubOrHexToPublicKeyBinary(admin); chk.E(e) { if a, e := bech32encoding.NpubOrHexToPublicKeyBinary(admin); chk.E(e) {
@ -114,39 +114,36 @@ func (f *Follows) Configure(cfg ...any) (err error) {
} }
newAdmins = append(newAdmins, adm) newAdmins = append(newAdmins, adm)
newAdminsSet[hex.EncodeToString(adm)] = struct{}{} newAdminsSet[hex.EncodeToString(adm)] = struct{}{}
adminBinaries = append(adminBinaries, adm)
}
// Batch query all admin follow lists in a single DB call
// Kind 3 is replaceable, so QueryEvents returns only the latest per author
if len(adminBinaries) > 0 {
ctx := f.Ctx
if ctx == nil {
ctx = context.Background()
}
fl := &filter.F{ fl := &filter.F{
Authors: tag.NewFromAny(adm), Authors: tag.NewFromBytesSlice(adminBinaries...),
Kinds: kind.NewS(kind.New(kind.FollowList.K)), Kinds: kind.NewS(kind.New(kind.FollowList.K)),
Limit: values.ToUintPointer(uint(len(adminBinaries))),
} }
var idxs []database.Range var evs event.S
if idxs, err = database.GetIndexesFromFilter(fl); chk.E(err) { if evs, err = f.db.QueryEvents(ctx, fl); err != nil {
return log.W.F("follows ACL: error querying admin follow lists: %v", err)
} err = nil // Don't fail Configure on query error
var sers types.Uint40s
for _, idx := range idxs {
var s types.Uint40s
if s, err = f.db.GetSerialsByRange(idx); chk.E(err) {
continue
}
sers = append(sers, s...)
} }
if len(sers) > 0 { for _, ev := range evs {
for _, s := range sers { for _, v := range ev.Tags.GetAll([]byte("p")) {
var ev *event.E // ValueHex() automatically handles both binary and hex storage formats
if ev, err = f.db.FetchEventBySerial(s); chk.E(err) { if b, e := hex.DecodeString(string(v.ValueHex())); chk.E(e) {
continue continue
} } else {
for _, v := range ev.Tags.GetAll([]byte("p")) { hexKey := hex.EncodeToString(b)
// ValueHex() automatically handles both binary and hex storage formats if _, exists := newFollowsSet[hexKey]; !exists {
if b, e := hex.DecodeString(string(v.ValueHex())); chk.E(e) { newFollows = append(newFollows, b)
continue newFollowsSet[hexKey] = struct{}{}
} else {
hexKey := hex.EncodeToString(b)
if _, exists := newFollowsSet[hexKey]; !exists {
newFollows = append(newFollows, b)
newFollowsSet[hexKey] = struct{}{}
}
} }
} }
} }
@ -294,29 +291,23 @@ func (f *Follows) adminRelays() (urls []string) {
} }
} }
// First, try to get relay URLs from admin kind 10002 events // Batch query all admin relay list events in a single DB call
for _, adm := range admins { // Kind 10002 is replaceable, so QueryEvents returns only the latest per author
if len(admins) > 0 {
ctx := f.Ctx
if ctx == nil {
ctx = context.Background()
}
fl := &filter.F{ fl := &filter.F{
Authors: tag.NewFromAny(adm), Authors: tag.NewFromBytesSlice(admins...),
Kinds: kind.NewS(kind.New(kind.RelayListMetadata.K)), Kinds: kind.NewS(kind.New(kind.RelayListMetadata.K)),
Limit: values.ToUintPointer(uint(len(admins))),
} }
idxs, err := database.GetIndexesFromFilter(fl) evs, qerr := f.db.QueryEvents(ctx, fl)
if chk.E(err) { if qerr != nil {
continue log.W.F("follows ACL: error querying admin relay lists: %v", qerr)
} }
var sers types.Uint40s for _, ev := range evs {
for _, idx := range idxs {
s, err := f.db.GetSerialsByRange(idx)
if chk.E(err) {
continue
}
sers = append(sers, s...)
}
for _, s := range sers {
ev, err := f.db.FetchEventBySerial(s)
if chk.E(err) || ev == nil {
continue
}
for _, v := range ev.Tags.GetAll([]byte("r")) { for _, v := range ev.Tags.GetAll([]byte("r")) {
u := string(v.Value()) u := string(v.Value())
n := string(normalize.URL(u)) n := string(normalize.URL(u))

2
pkg/version/version

@ -1 +1 @@
v0.58.5 v0.58.6

Loading…
Cancel
Save