Browse Source
Major refactoring to support running the database as a separate gRPC service: - Add gRPC database server (cmd/orly-db/) and launcher (cmd/orly-launcher/) - Add proto definitions for all database operations (proto/orlydb/v1/) - Add gRPC client implementing database.Database interface - Abstract Blossom blob storage into database interface (9 methods) - Remove BBolt database backend (pkg/bbolt/ deleted) - Hide Blossom tab in UI when blob storage unavailable - Update documentation for new architecture Database backends: badger (default), neo4j, wasmdb, grpc Files modified: - cmd/orly-db/: New gRPC database server - cmd/orly-launcher/: Process supervisor for split mode - proto/orlydb/v1/: Protocol buffer definitions - pkg/database/grpc/: gRPC client implementation - pkg/database/blob.go: Badger blob storage implementation - pkg/database/interface.go: Added blob storage methods - pkg/database/types.go: Added BlobMetadata, BlobDescriptor types - pkg/database/factory.go: Removed BBolt, added gRPC backend - pkg/neo4j/blob.go: Blob storage stubs - pkg/wasmdb/blob.go: Blob storage stubs - pkg/blossom/: Refactored to use database.Database interface - app/config/config.go: Removed BBolt config, added gRPC settings - CLAUDE.md: Updated documentation Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>main v0.53.0
woikos
4 months ago
57 changed files with 13766 additions and 4108 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,68 @@
@@ -0,0 +1,68 @@
|
||||
package main |
||||
|
||||
import ( |
||||
"os" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"go-simpler.org/env" |
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
) |
||||
|
||||
// Config holds the database server configuration.
|
||||
type Config struct { |
||||
// Listen is the gRPC server listen address
|
||||
Listen string `env:"ORLY_DB_LISTEN" default:"127.0.0.1:50051" usage:"gRPC server listen address"` |
||||
|
||||
// DataDir is the database data directory
|
||||
DataDir string `env:"ORLY_DATA_DIR" usage:"database data directory"` |
||||
|
||||
// LogLevel is the logging level
|
||||
LogLevel string `env:"ORLY_DB_LOG_LEVEL" default:"info" usage:"log level (trace, debug, info, warn, error)"` |
||||
|
||||
// Badger configuration
|
||||
BlockCacheMB int `env:"ORLY_DB_BLOCK_CACHE_MB" default:"1024" usage:"block cache size in MB"` |
||||
IndexCacheMB int `env:"ORLY_DB_INDEX_CACHE_MB" default:"512" usage:"index cache size in MB"` |
||||
ZSTDLevel int `env:"ORLY_DB_ZSTD_LEVEL" default:"3" usage:"ZSTD compression level (1-19)"` |
||||
|
||||
// Query cache configuration
|
||||
QueryCacheSizeMB int `env:"ORLY_DB_QUERY_CACHE_SIZE_MB" default:"256" usage:"query cache size in MB"` |
||||
QueryCacheMaxAge time.Duration `env:"ORLY_DB_QUERY_CACHE_MAX_AGE" default:"5m" usage:"query cache max age"` |
||||
QueryCacheDisabled bool `env:"ORLY_DB_QUERY_CACHE_DISABLED" default:"false" usage:"disable query cache"` |
||||
|
||||
// Serial cache configuration
|
||||
SerialCachePubkeys int `env:"ORLY_SERIAL_CACHE_PUBKEYS" default:"100000" usage:"serial cache pubkeys capacity"` |
||||
SerialCacheEventIds int `env:"ORLY_SERIAL_CACHE_EVENT_IDS" default:"500000" usage:"serial cache event IDs capacity"` |
||||
|
||||
// gRPC server configuration
|
||||
MaxConcurrentQueries int `env:"ORLY_DB_MAX_CONCURRENT_QUERIES" default:"3" usage:"max concurrent queries"` |
||||
StreamBatchSize int `env:"ORLY_DB_STREAM_BATCH_SIZE" default:"100" usage:"events per stream batch"` |
||||
} |
||||
|
||||
// loadConfig loads configuration from environment variables.
|
||||
func loadConfig() *Config { |
||||
cfg := &Config{} |
||||
if err := env.Load(cfg, nil); chk.E(err) { |
||||
log.E.F("failed to load config: %v", err) |
||||
os.Exit(1) |
||||
} |
||||
|
||||
// Set default data directory if not specified
|
||||
if cfg.DataDir == "" { |
||||
home, err := os.UserHomeDir() |
||||
if chk.E(err) { |
||||
log.E.F("failed to get home directory: %v", err) |
||||
os.Exit(1) |
||||
} |
||||
cfg.DataDir = filepath.Join(home, ".local", "share", "ORLY") |
||||
} |
||||
|
||||
// Ensure data directory exists
|
||||
if err := os.MkdirAll(cfg.DataDir, 0700); chk.E(err) { |
||||
log.E.F("failed to create data directory %s: %v", cfg.DataDir, err) |
||||
os.Exit(1) |
||||
} |
||||
|
||||
return cfg |
||||
} |
||||
@ -0,0 +1,122 @@
@@ -0,0 +1,122 @@
|
||||
// orly-db is a standalone gRPC database server for the ORLY relay.
|
||||
// It wraps the Badger database implementation and exposes it via gRPC.
|
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"net" |
||||
"os" |
||||
"os/signal" |
||||
"syscall" |
||||
"time" |
||||
|
||||
"google.golang.org/grpc" |
||||
"google.golang.org/grpc/reflection" |
||||
"lol.mleku.dev" |
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
|
||||
"next.orly.dev/pkg/database" |
||||
orlydbv1 "next.orly.dev/pkg/proto/orlydb/v1" |
||||
) |
||||
|
||||
func main() { |
||||
cfg := loadConfig() |
||||
|
||||
// Set log level
|
||||
lol.SetLogLevel(cfg.LogLevel) |
||||
log.I.F("orly-db starting with log level: %s", cfg.LogLevel) |
||||
|
||||
ctx, cancel := context.WithCancel(context.Background()) |
||||
defer cancel() |
||||
|
||||
// Create database configuration
|
||||
dbCfg := &database.DatabaseConfig{ |
||||
DataDir: cfg.DataDir, |
||||
LogLevel: cfg.LogLevel, |
||||
BlockCacheMB: cfg.BlockCacheMB, |
||||
IndexCacheMB: cfg.IndexCacheMB, |
||||
QueryCacheSizeMB: cfg.QueryCacheSizeMB, |
||||
QueryCacheMaxAge: cfg.QueryCacheMaxAge, |
||||
QueryCacheDisabled: cfg.QueryCacheDisabled, |
||||
SerialCachePubkeys: cfg.SerialCachePubkeys, |
||||
SerialCacheEventIds: cfg.SerialCacheEventIds, |
||||
ZSTDLevel: cfg.ZSTDLevel, |
||||
} |
||||
|
||||
// Initialize database using existing Badger implementation
|
||||
log.I.F("initializing database at %s", cfg.DataDir) |
||||
db, err := database.NewWithConfig(ctx, cancel, dbCfg) |
||||
if chk.E(err) { |
||||
log.E.F("failed to initialize database: %v", err) |
||||
os.Exit(1) |
||||
} |
||||
|
||||
// Wait for database to be ready
|
||||
log.I.F("waiting for database to be ready...") |
||||
<-db.Ready() |
||||
log.I.F("database ready") |
||||
|
||||
// Create gRPC server with large message sizes for events
|
||||
grpcServer := grpc.NewServer( |
||||
grpc.MaxRecvMsgSize(64<<20), // 64MB
|
||||
grpc.MaxSendMsgSize(64<<20), // 64MB
|
||||
) |
||||
|
||||
// Register database service
|
||||
service := NewDatabaseService(db, cfg) |
||||
orlydbv1.RegisterDatabaseServiceServer(grpcServer, service) |
||||
|
||||
// Register reflection for debugging with grpcurl
|
||||
reflection.Register(grpcServer) |
||||
|
||||
// Start listening
|
||||
lis, err := net.Listen("tcp", cfg.Listen) |
||||
if chk.E(err) { |
||||
log.E.F("failed to listen on %s: %v", cfg.Listen, err) |
||||
os.Exit(1) |
||||
} |
||||
log.I.F("gRPC server listening on %s", cfg.Listen) |
||||
|
||||
// Handle graceful shutdown
|
||||
go func() { |
||||
sigs := make(chan os.Signal, 1) |
||||
signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) |
||||
sig := <-sigs |
||||
log.I.F("received signal %v, shutting down...", sig) |
||||
|
||||
// Cancel context to stop all operations
|
||||
cancel() |
||||
|
||||
// Gracefully stop gRPC server with timeout
|
||||
stopped := make(chan struct{}) |
||||
go func() { |
||||
grpcServer.GracefulStop() |
||||
close(stopped) |
||||
}() |
||||
|
||||
select { |
||||
case <-stopped: |
||||
log.I.F("gRPC server stopped gracefully") |
||||
case <-time.After(5 * time.Second): |
||||
log.W.F("gRPC graceful stop timed out, forcing stop") |
||||
grpcServer.Stop() |
||||
} |
||||
|
||||
// Sync and close database
|
||||
log.I.F("syncing database...") |
||||
if err := db.Sync(); chk.E(err) { |
||||
log.W.F("failed to sync database: %v", err) |
||||
} |
||||
log.I.F("closing database...") |
||||
if err := db.Close(); chk.E(err) { |
||||
log.W.F("failed to close database: %v", err) |
||||
} |
||||
log.I.F("shutdown complete") |
||||
}() |
||||
|
||||
// Serve gRPC
|
||||
if err := grpcServer.Serve(lis); err != nil { |
||||
log.E.F("gRPC server error: %v", err) |
||||
} |
||||
} |
||||
@ -0,0 +1,731 @@
@@ -0,0 +1,731 @@
|
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
|
||||
"google.golang.org/grpc/codes" |
||||
"google.golang.org/grpc/status" |
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
|
||||
"next.orly.dev/pkg/database" |
||||
orlydbv1 "next.orly.dev/pkg/proto/orlydb/v1" |
||||
) |
||||
|
||||
// DatabaseService implements the orlydbv1.DatabaseServiceServer interface.
|
||||
type DatabaseService struct { |
||||
orlydbv1.UnimplementedDatabaseServiceServer |
||||
db database.Database |
||||
cfg *Config |
||||
} |
||||
|
||||
// NewDatabaseService creates a new database service.
|
||||
func NewDatabaseService(db database.Database, cfg *Config) *DatabaseService { |
||||
return &DatabaseService{ |
||||
db: db, |
||||
cfg: cfg, |
||||
} |
||||
} |
||||
|
||||
// === Lifecycle Methods ===
|
||||
|
||||
func (s *DatabaseService) GetPath(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.PathResponse, error) { |
||||
return &orlydbv1.PathResponse{Path: s.db.Path()}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) Sync(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.Empty, error) { |
||||
if err := s.db.Sync(); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "sync failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) Ready(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.ReadyResponse, error) { |
||||
// Check if ready channel is closed
|
||||
select { |
||||
case <-s.db.Ready(): |
||||
return &orlydbv1.ReadyResponse{Ready: true}, nil |
||||
default: |
||||
return &orlydbv1.ReadyResponse{Ready: false}, nil |
||||
} |
||||
} |
||||
|
||||
func (s *DatabaseService) SetLogLevel(ctx context.Context, req *orlydbv1.SetLogLevelRequest) (*orlydbv1.Empty, error) { |
||||
s.db.SetLogLevel(req.Level) |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
// === Event Storage ===
|
||||
|
||||
func (s *DatabaseService) SaveEvent(ctx context.Context, req *orlydbv1.SaveEventRequest) (*orlydbv1.SaveEventResponse, error) { |
||||
ev := orlydbv1.ProtoToEvent(req.Event) |
||||
exists, err := s.db.SaveEvent(ctx, ev) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "save event failed: %v", err) |
||||
} |
||||
return &orlydbv1.SaveEventResponse{Exists: exists}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetSerialsFromFilter(ctx context.Context, req *orlydbv1.GetSerialsFromFilterRequest) (*orlydbv1.SerialList, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
serials, err := s.db.GetSerialsFromFilter(f) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get serials failed: %v", err) |
||||
} |
||||
return orlydbv1.Uint40sToProto(serials), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) WouldReplaceEvent(ctx context.Context, req *orlydbv1.WouldReplaceEventRequest) (*orlydbv1.WouldReplaceEventResponse, error) { |
||||
ev := orlydbv1.ProtoToEvent(req.Event) |
||||
wouldReplace, replacedSerials, err := s.db.WouldReplaceEvent(ev) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "would replace check failed: %v", err) |
||||
} |
||||
resp := &orlydbv1.WouldReplaceEventResponse{ |
||||
WouldReplace: wouldReplace, |
||||
} |
||||
for _, ser := range replacedSerials { |
||||
resp.ReplacedSerials = append(resp.ReplacedSerials, ser.Get()) |
||||
} |
||||
return resp, nil |
||||
} |
||||
|
||||
// === Event Queries (Streaming) ===
|
||||
|
||||
func (s *DatabaseService) QueryEvents(req *orlydbv1.QueryEventsRequest, stream orlydbv1.DatabaseService_QueryEventsServer) error { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
events, err := s.db.QueryEvents(stream.Context(), f) |
||||
if err != nil { |
||||
return status.Errorf(codes.Internal, "query events failed: %v", err) |
||||
} |
||||
return s.streamEvents(orlydbv1.EventsToProto(events), stream) |
||||
} |
||||
|
||||
func (s *DatabaseService) QueryAllVersions(req *orlydbv1.QueryEventsRequest, stream orlydbv1.DatabaseService_QueryAllVersionsServer) error { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
events, err := s.db.QueryAllVersions(stream.Context(), f) |
||||
if err != nil { |
||||
return status.Errorf(codes.Internal, "query all versions failed: %v", err) |
||||
} |
||||
return s.streamEvents(orlydbv1.EventsToProto(events), stream) |
||||
} |
||||
|
||||
func (s *DatabaseService) QueryEventsWithOptions(req *orlydbv1.QueryEventsWithOptionsRequest, stream orlydbv1.DatabaseService_QueryEventsWithOptionsServer) error { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
events, err := s.db.QueryEventsWithOptions(stream.Context(), f, req.IncludeDeleteEvents, req.ShowAllVersions) |
||||
if err != nil { |
||||
return status.Errorf(codes.Internal, "query events with options failed: %v", err) |
||||
} |
||||
return s.streamEvents(orlydbv1.EventsToProto(events), stream) |
||||
} |
||||
|
||||
func (s *DatabaseService) QueryDeleteEventsByTargetId(req *orlydbv1.QueryDeleteEventsByTargetIdRequest, stream orlydbv1.DatabaseService_QueryDeleteEventsByTargetIdServer) error { |
||||
events, err := s.db.QueryDeleteEventsByTargetId(stream.Context(), req.TargetEventId) |
||||
if err != nil { |
||||
return status.Errorf(codes.Internal, "query delete events failed: %v", err) |
||||
} |
||||
return s.streamEvents(orlydbv1.EventsToProto(events), stream) |
||||
} |
||||
|
||||
func (s *DatabaseService) QueryForSerials(ctx context.Context, req *orlydbv1.QueryEventsRequest) (*orlydbv1.SerialList, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
serials, err := s.db.QueryForSerials(ctx, f) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "query for serials failed: %v", err) |
||||
} |
||||
return orlydbv1.Uint40sToProto(serials), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) QueryForIds(ctx context.Context, req *orlydbv1.QueryEventsRequest) (*orlydbv1.IdPkTsList, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
idPkTs, err := s.db.QueryForIds(ctx, f) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "query for ids failed: %v", err) |
||||
} |
||||
return orlydbv1.IdPkTsListToProto(idPkTs), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) CountEvents(ctx context.Context, req *orlydbv1.QueryEventsRequest) (*orlydbv1.CountEventsResponse, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
count, approximate, err := s.db.CountEvents(ctx, f) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "count events failed: %v", err) |
||||
} |
||||
return &orlydbv1.CountEventsResponse{ |
||||
Count: int32(count), |
||||
Approximate: approximate, |
||||
}, nil |
||||
} |
||||
|
||||
// === Event Retrieval by Serial ===
|
||||
|
||||
func (s *DatabaseService) FetchEventBySerial(ctx context.Context, req *orlydbv1.FetchEventBySerialRequest) (*orlydbv1.FetchEventBySerialResponse, error) { |
||||
ser := orlydbv1.ProtoToUint40(&orlydbv1.Uint40{Value: req.Serial}) |
||||
ev, err := s.db.FetchEventBySerial(ser) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "fetch event by serial failed: %v", err) |
||||
} |
||||
return &orlydbv1.FetchEventBySerialResponse{ |
||||
Event: orlydbv1.EventToProto(ev), |
||||
Found: ev != nil, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) FetchEventsBySerials(ctx context.Context, req *orlydbv1.FetchEventsBySerialRequest) (*orlydbv1.EventMap, error) { |
||||
serials := orlydbv1.ProtoToUint40s(&orlydbv1.SerialList{Serials: req.Serials}) |
||||
events, err := s.db.FetchEventsBySerials(serials) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "fetch events by serials failed: %v", err) |
||||
} |
||||
return orlydbv1.EventMapToProto(events), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetSerialById(ctx context.Context, req *orlydbv1.GetSerialByIdRequest) (*orlydbv1.GetSerialByIdResponse, error) { |
||||
ser, err := s.db.GetSerialById(req.Id) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get serial by id failed: %v", err) |
||||
} |
||||
if ser == nil { |
||||
return &orlydbv1.GetSerialByIdResponse{Found: false}, nil |
||||
} |
||||
return &orlydbv1.GetSerialByIdResponse{ |
||||
Serial: ser.Get(), |
||||
Found: true, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetSerialsByIds(ctx context.Context, req *orlydbv1.GetSerialsByIdsRequest) (*orlydbv1.SerialMap, error) { |
||||
// Convert request IDs to tag format
|
||||
ids := orlydbv1.BytesToTag(req.Ids) |
||||
serials, err := s.db.GetSerialsByIds(ids) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get serials by ids failed: %v", err) |
||||
} |
||||
result := &orlydbv1.SerialMap{ |
||||
Serials: make(map[string]uint64), |
||||
} |
||||
for k, v := range serials { |
||||
if v != nil { |
||||
result.Serials[k] = v.Get() |
||||
} |
||||
} |
||||
return result, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetSerialsByRange(ctx context.Context, req *orlydbv1.GetSerialsByRangeRequest) (*orlydbv1.SerialList, error) { |
||||
r := orlydbv1.ProtoToRange(req.Range) |
||||
serials, err := s.db.GetSerialsByRange(r) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get serials by range failed: %v", err) |
||||
} |
||||
return orlydbv1.Uint40sToProto(serials), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetFullIdPubkeyBySerial(ctx context.Context, req *orlydbv1.GetFullIdPubkeyBySerialRequest) (*orlydbv1.IdPkTs, error) { |
||||
ser := orlydbv1.ProtoToUint40(&orlydbv1.Uint40{Value: req.Serial}) |
||||
idPkTs, err := s.db.GetFullIdPubkeyBySerial(ser) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get full id pubkey by serial failed: %v", err) |
||||
} |
||||
return orlydbv1.IdPkTsToProto(idPkTs), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetFullIdPubkeyBySerials(ctx context.Context, req *orlydbv1.GetFullIdPubkeyBySerialsRequest) (*orlydbv1.IdPkTsList, error) { |
||||
serials := orlydbv1.ProtoToUint40s(&orlydbv1.SerialList{Serials: req.Serials}) |
||||
idPkTs, err := s.db.GetFullIdPubkeyBySerials(serials) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get full id pubkey by serials failed: %v", err) |
||||
} |
||||
return orlydbv1.IdPkTsListToProto(idPkTs), nil |
||||
} |
||||
|
||||
// === Event Deletion ===
|
||||
|
||||
func (s *DatabaseService) DeleteEvent(ctx context.Context, req *orlydbv1.DeleteEventRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.DeleteEvent(ctx, req.EventId); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "delete event failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) DeleteEventBySerial(ctx context.Context, req *orlydbv1.DeleteEventBySerialRequest) (*orlydbv1.Empty, error) { |
||||
ser := orlydbv1.ProtoToUint40(&orlydbv1.Uint40{Value: req.Serial}) |
||||
ev := orlydbv1.ProtoToEvent(req.Event) |
||||
if err := s.db.DeleteEventBySerial(ctx, ser, ev); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "delete event by serial failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) DeleteExpired(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.Empty, error) { |
||||
s.db.DeleteExpired() |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) ProcessDelete(ctx context.Context, req *orlydbv1.ProcessDeleteRequest) (*orlydbv1.Empty, error) { |
||||
ev := orlydbv1.ProtoToEvent(req.Event) |
||||
if err := s.db.ProcessDelete(ev, req.Admins); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "process delete failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) CheckForDeleted(ctx context.Context, req *orlydbv1.CheckForDeletedRequest) (*orlydbv1.Empty, error) { |
||||
ev := orlydbv1.ProtoToEvent(req.Event) |
||||
if err := s.db.CheckForDeleted(ev, req.Admins); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "check for deleted failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
// === Import/Export ===
|
||||
|
||||
func (s *DatabaseService) Import(stream orlydbv1.DatabaseService_ImportServer) error { |
||||
pr, pw := io.Pipe() |
||||
|
||||
// Goroutine to read from gRPC stream and write to pipe
|
||||
go func() { |
||||
defer pw.Close() |
||||
for { |
||||
chunk, err := stream.Recv() |
||||
if err == io.EOF { |
||||
return |
||||
} |
||||
if err != nil { |
||||
log.E.F("import stream error: %v", err) |
||||
pw.CloseWithError(err) |
||||
return |
||||
} |
||||
if _, err := pw.Write(chunk.Data); chk.E(err) { |
||||
return |
||||
} |
||||
} |
||||
}() |
||||
|
||||
// Import from pipe
|
||||
s.db.Import(pr) |
||||
|
||||
return stream.SendAndClose(&orlydbv1.ImportResponse{ |
||||
EventsImported: 0, // TODO: Track count
|
||||
EventsSkipped: 0, |
||||
}) |
||||
} |
||||
|
||||
func (s *DatabaseService) Export(req *orlydbv1.ExportRequest, stream orlydbv1.DatabaseService_ExportServer) error { |
||||
pr, pw := io.Pipe() |
||||
|
||||
// Goroutine to export to pipe
|
||||
go func() { |
||||
defer pw.Close() |
||||
s.db.Export(stream.Context(), pw, req.Pubkeys...) |
||||
}() |
||||
|
||||
// Read from pipe and send to stream
|
||||
buf := make([]byte, 64*1024) // 64KB chunks
|
||||
for { |
||||
n, err := pr.Read(buf) |
||||
if err == io.EOF { |
||||
return nil |
||||
} |
||||
if err != nil { |
||||
return status.Errorf(codes.Internal, "export failed: %v", err) |
||||
} |
||||
if err := stream.Send(&orlydbv1.ExportChunk{Data: buf[:n]}); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (s *DatabaseService) ImportEventsFromStrings(ctx context.Context, req *orlydbv1.ImportEventsFromStringsRequest) (*orlydbv1.ImportResponse, error) { |
||||
// Note: We can't pass policy manager over gRPC, so we pass nil
|
||||
if err := s.db.ImportEventsFromStrings(ctx, req.EventJsons, nil); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "import events from strings failed: %v", err) |
||||
} |
||||
return &orlydbv1.ImportResponse{ |
||||
EventsImported: int64(len(req.EventJsons)), |
||||
}, nil |
||||
} |
||||
|
||||
// === Relay Identity ===
|
||||
|
||||
func (s *DatabaseService) GetRelayIdentitySecret(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.GetRelayIdentitySecretResponse, error) { |
||||
secret, err := s.db.GetRelayIdentitySecret() |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get relay identity secret failed: %v", err) |
||||
} |
||||
return &orlydbv1.GetRelayIdentitySecretResponse{SecretKey: secret}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) SetRelayIdentitySecret(ctx context.Context, req *orlydbv1.SetRelayIdentitySecretRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.SetRelayIdentitySecret(req.SecretKey); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "set relay identity secret failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetOrCreateRelayIdentitySecret(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.GetRelayIdentitySecretResponse, error) { |
||||
secret, err := s.db.GetOrCreateRelayIdentitySecret() |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get or create relay identity secret failed: %v", err) |
||||
} |
||||
return &orlydbv1.GetRelayIdentitySecretResponse{SecretKey: secret}, nil |
||||
} |
||||
|
||||
// === Markers ===
|
||||
|
||||
func (s *DatabaseService) SetMarker(ctx context.Context, req *orlydbv1.SetMarkerRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.SetMarker(req.Key, req.Value); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "set marker failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetMarker(ctx context.Context, req *orlydbv1.GetMarkerRequest) (*orlydbv1.GetMarkerResponse, error) { |
||||
value, err := s.db.GetMarker(req.Key) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get marker failed: %v", err) |
||||
} |
||||
return &orlydbv1.GetMarkerResponse{ |
||||
Value: value, |
||||
Found: value != nil, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) HasMarker(ctx context.Context, req *orlydbv1.HasMarkerRequest) (*orlydbv1.HasMarkerResponse, error) { |
||||
exists := s.db.HasMarker(req.Key) |
||||
return &orlydbv1.HasMarkerResponse{Exists: exists}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) DeleteMarker(ctx context.Context, req *orlydbv1.DeleteMarkerRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.DeleteMarker(req.Key); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "delete marker failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
// === Subscriptions ===
|
||||
|
||||
func (s *DatabaseService) GetSubscription(ctx context.Context, req *orlydbv1.GetSubscriptionRequest) (*orlydbv1.Subscription, error) { |
||||
sub, err := s.db.GetSubscription(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get subscription failed: %v", err) |
||||
} |
||||
return orlydbv1.SubscriptionToProto(sub, req.Pubkey), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) IsSubscriptionActive(ctx context.Context, req *orlydbv1.IsSubscriptionActiveRequest) (*orlydbv1.IsSubscriptionActiveResponse, error) { |
||||
active, err := s.db.IsSubscriptionActive(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "is subscription active failed: %v", err) |
||||
} |
||||
return &orlydbv1.IsSubscriptionActiveResponse{Active: active}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) ExtendSubscription(ctx context.Context, req *orlydbv1.ExtendSubscriptionRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.ExtendSubscription(req.Pubkey, int(req.Days)); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "extend subscription failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) RecordPayment(ctx context.Context, req *orlydbv1.RecordPaymentRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.RecordPayment(req.Pubkey, req.Amount, req.Invoice, req.Preimage); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "record payment failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetPaymentHistory(ctx context.Context, req *orlydbv1.GetPaymentHistoryRequest) (*orlydbv1.PaymentList, error) { |
||||
payments, err := s.db.GetPaymentHistory(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get payment history failed: %v", err) |
||||
} |
||||
return orlydbv1.PaymentListToProto(payments), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) ExtendBlossomSubscription(ctx context.Context, req *orlydbv1.ExtendBlossomSubscriptionRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.ExtendBlossomSubscription(req.Pubkey, req.Tier, req.StorageMb, int(req.DaysExtended)); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "extend blossom subscription failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetBlossomStorageQuota(ctx context.Context, req *orlydbv1.GetBlossomStorageQuotaRequest) (*orlydbv1.GetBlossomStorageQuotaResponse, error) { |
||||
quota, err := s.db.GetBlossomStorageQuota(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get blossom storage quota failed: %v", err) |
||||
} |
||||
return &orlydbv1.GetBlossomStorageQuotaResponse{QuotaMb: quota}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) IsFirstTimeUser(ctx context.Context, req *orlydbv1.IsFirstTimeUserRequest) (*orlydbv1.IsFirstTimeUserResponse, error) { |
||||
firstTime, err := s.db.IsFirstTimeUser(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "is first time user failed: %v", err) |
||||
} |
||||
return &orlydbv1.IsFirstTimeUserResponse{FirstTime: firstTime}, nil |
||||
} |
||||
|
||||
// === NIP-43 ===
|
||||
|
||||
func (s *DatabaseService) AddNIP43Member(ctx context.Context, req *orlydbv1.AddNIP43MemberRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.AddNIP43Member(req.Pubkey, req.InviteCode); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "add NIP-43 member failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) RemoveNIP43Member(ctx context.Context, req *orlydbv1.RemoveNIP43MemberRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.RemoveNIP43Member(req.Pubkey); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "remove NIP-43 member failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) IsNIP43Member(ctx context.Context, req *orlydbv1.IsNIP43MemberRequest) (*orlydbv1.IsNIP43MemberResponse, error) { |
||||
isMember, err := s.db.IsNIP43Member(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "is NIP-43 member failed: %v", err) |
||||
} |
||||
return &orlydbv1.IsNIP43MemberResponse{IsMember: isMember}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetNIP43Membership(ctx context.Context, req *orlydbv1.GetNIP43MembershipRequest) (*orlydbv1.NIP43Membership, error) { |
||||
membership, err := s.db.GetNIP43Membership(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get NIP-43 membership failed: %v", err) |
||||
} |
||||
return orlydbv1.NIP43MembershipToProto(membership), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetAllNIP43Members(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.PubkeyList, error) { |
||||
members, err := s.db.GetAllNIP43Members() |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get all NIP-43 members failed: %v", err) |
||||
} |
||||
return &orlydbv1.PubkeyList{Pubkeys: members}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) StoreInviteCode(ctx context.Context, req *orlydbv1.StoreInviteCodeRequest) (*orlydbv1.Empty, error) { |
||||
expiresAt := orlydbv1.TimeFromUnix(req.ExpiresAt) |
||||
if err := s.db.StoreInviteCode(req.Code, expiresAt); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "store invite code failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) ValidateInviteCode(ctx context.Context, req *orlydbv1.ValidateInviteCodeRequest) (*orlydbv1.ValidateInviteCodeResponse, error) { |
||||
valid, err := s.db.ValidateInviteCode(req.Code) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "validate invite code failed: %v", err) |
||||
} |
||||
return &orlydbv1.ValidateInviteCodeResponse{Valid: valid}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) DeleteInviteCode(ctx context.Context, req *orlydbv1.DeleteInviteCodeRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.DeleteInviteCode(req.Code); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "delete invite code failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) PublishNIP43MembershipEvent(ctx context.Context, req *orlydbv1.PublishNIP43MembershipEventRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.PublishNIP43MembershipEvent(int(req.Kind), req.Pubkey); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "publish NIP-43 membership event failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
// === Query Cache ===
|
||||
|
||||
func (s *DatabaseService) GetCachedJSON(ctx context.Context, req *orlydbv1.GetCachedJSONRequest) (*orlydbv1.GetCachedJSONResponse, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
jsonItems, found := s.db.GetCachedJSON(f) |
||||
return &orlydbv1.GetCachedJSONResponse{ |
||||
JsonItems: jsonItems, |
||||
Found: found, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) CacheMarshaledJSON(ctx context.Context, req *orlydbv1.CacheMarshaledJSONRequest) (*orlydbv1.Empty, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
s.db.CacheMarshaledJSON(f, req.JsonItems) |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetCachedEvents(ctx context.Context, req *orlydbv1.GetCachedEventsRequest) (*orlydbv1.GetCachedEventsResponse, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
events, found := s.db.GetCachedEvents(f) |
||||
return &orlydbv1.GetCachedEventsResponse{ |
||||
Events: orlydbv1.EventsToProto(events), |
||||
Found: found, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) CacheEvents(ctx context.Context, req *orlydbv1.CacheEventsRequest) (*orlydbv1.Empty, error) { |
||||
f := orlydbv1.ProtoToFilter(req.Filter) |
||||
events := orlydbv1.ProtoToEvents(req.Events) |
||||
s.db.CacheEvents(f, events) |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) InvalidateQueryCache(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.Empty, error) { |
||||
s.db.InvalidateQueryCache() |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
// === Access Tracking ===
|
||||
|
||||
func (s *DatabaseService) RecordEventAccess(ctx context.Context, req *orlydbv1.RecordEventAccessRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.RecordEventAccess(req.Serial, req.ConnectionId); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "record event access failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetEventAccessInfo(ctx context.Context, req *orlydbv1.GetEventAccessInfoRequest) (*orlydbv1.GetEventAccessInfoResponse, error) { |
||||
lastAccess, accessCount, err := s.db.GetEventAccessInfo(req.Serial) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get event access info failed: %v", err) |
||||
} |
||||
return &orlydbv1.GetEventAccessInfoResponse{ |
||||
LastAccess: lastAccess, |
||||
AccessCount: accessCount, |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetLeastAccessedEvents(ctx context.Context, req *orlydbv1.GetLeastAccessedEventsRequest) (*orlydbv1.SerialList, error) { |
||||
serials, err := s.db.GetLeastAccessedEvents(int(req.Limit), req.MinAgeSec) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get least accessed events failed: %v", err) |
||||
} |
||||
return &orlydbv1.SerialList{Serials: serials}, nil |
||||
} |
||||
|
||||
// === Utility ===
|
||||
|
||||
func (s *DatabaseService) EventIdsBySerial(ctx context.Context, req *orlydbv1.EventIdsBySerialRequest) (*orlydbv1.EventIdsBySerialResponse, error) { |
||||
eventIds, err := s.db.EventIdsBySerial(req.Start, int(req.Count)) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "event ids by serial failed: %v", err) |
||||
} |
||||
return &orlydbv1.EventIdsBySerialResponse{EventIds: eventIds}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) RunMigrations(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.Empty, error) { |
||||
s.db.RunMigrations() |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
// === Blob Storage (Blossom) ===
|
||||
|
||||
func (s *DatabaseService) SaveBlob(ctx context.Context, req *orlydbv1.SaveBlobRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.SaveBlob(req.Sha256Hash, req.Data, req.Pubkey, req.MimeType, req.Extension); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "save blob failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetBlob(ctx context.Context, req *orlydbv1.GetBlobRequest) (*orlydbv1.GetBlobResponse, error) { |
||||
data, metadata, err := s.db.GetBlob(req.Sha256Hash) |
||||
if err != nil { |
||||
// Return not found as a response, not an error
|
||||
return &orlydbv1.GetBlobResponse{Found: false}, nil |
||||
} |
||||
return &orlydbv1.GetBlobResponse{ |
||||
Found: true, |
||||
Data: data, |
||||
Metadata: orlydbv1.BlobMetadataToProto(metadata), |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) HasBlob(ctx context.Context, req *orlydbv1.HasBlobRequest) (*orlydbv1.HasBlobResponse, error) { |
||||
exists, err := s.db.HasBlob(req.Sha256Hash) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "has blob failed: %v", err) |
||||
} |
||||
return &orlydbv1.HasBlobResponse{Exists: exists}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) DeleteBlob(ctx context.Context, req *orlydbv1.DeleteBlobRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.DeleteBlob(req.Sha256Hash, req.Pubkey); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "delete blob failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) ListBlobs(ctx context.Context, req *orlydbv1.ListBlobsRequest) (*orlydbv1.ListBlobsResponse, error) { |
||||
descriptors, err := s.db.ListBlobs(req.Pubkey, req.Since, req.Until) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "list blobs failed: %v", err) |
||||
} |
||||
return &orlydbv1.ListBlobsResponse{ |
||||
Descriptors: orlydbv1.BlobDescriptorListToProto(descriptors), |
||||
}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetBlobMetadata(ctx context.Context, req *orlydbv1.GetBlobMetadataRequest) (*orlydbv1.BlobMetadata, error) { |
||||
metadata, err := s.db.GetBlobMetadata(req.Sha256Hash) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.NotFound, "blob metadata not found: %v", err) |
||||
} |
||||
return orlydbv1.BlobMetadataToProto(metadata), nil |
||||
} |
||||
|
||||
func (s *DatabaseService) GetTotalBlobStorageUsed(ctx context.Context, req *orlydbv1.GetTotalBlobStorageUsedRequest) (*orlydbv1.GetTotalBlobStorageUsedResponse, error) { |
||||
totalMB, err := s.db.GetTotalBlobStorageUsed(req.Pubkey) |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "get total blob storage used failed: %v", err) |
||||
} |
||||
return &orlydbv1.GetTotalBlobStorageUsedResponse{TotalMb: totalMB}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) SaveBlobReport(ctx context.Context, req *orlydbv1.SaveBlobReportRequest) (*orlydbv1.Empty, error) { |
||||
if err := s.db.SaveBlobReport(req.Sha256Hash, req.ReportData); err != nil { |
||||
return nil, status.Errorf(codes.Internal, "save blob report failed: %v", err) |
||||
} |
||||
return &orlydbv1.Empty{}, nil |
||||
} |
||||
|
||||
func (s *DatabaseService) ListAllBlobUserStats(ctx context.Context, req *orlydbv1.Empty) (*orlydbv1.ListAllBlobUserStatsResponse, error) { |
||||
stats, err := s.db.ListAllBlobUserStats() |
||||
if err != nil { |
||||
return nil, status.Errorf(codes.Internal, "list all blob user stats failed: %v", err) |
||||
} |
||||
return &orlydbv1.ListAllBlobUserStatsResponse{ |
||||
Stats: orlydbv1.UserBlobStatsListToProto(stats), |
||||
}, nil |
||||
} |
||||
|
||||
// === Helper Methods ===
|
||||
|
||||
// streamEvents is a helper to stream events in batches.
|
||||
type eventStreamer interface { |
||||
Send(*orlydbv1.EventBatch) error |
||||
Context() context.Context |
||||
} |
||||
|
||||
func (s *DatabaseService) streamEvents(events []*orlydbv1.Event, stream eventStreamer) error { |
||||
batchSize := s.cfg.StreamBatchSize |
||||
if batchSize == 0 { |
||||
batchSize = 100 |
||||
} |
||||
|
||||
for i := 0; i < len(events); i += batchSize { |
||||
end := i + batchSize |
||||
if end > len(events) { |
||||
end = len(events) |
||||
} |
||||
|
||||
batch := &orlydbv1.EventBatch{ |
||||
Events: events[i:end], |
||||
} |
||||
|
||||
if err := stream.Send(batch); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,63 @@
@@ -0,0 +1,63 @@
|
||||
package main |
||||
|
||||
import ( |
||||
"os" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"github.com/adrg/xdg" |
||||
) |
||||
|
||||
// Config holds the launcher configuration.
|
||||
type Config struct { |
||||
// DBBinary is the path to the orly-db binary
|
||||
DBBinary string |
||||
|
||||
// RelayBinary is the path to the orly binary
|
||||
RelayBinary string |
||||
|
||||
// DBListen is the address the database server listens on
|
||||
DBListen string |
||||
|
||||
// DBReadyTimeout is how long to wait for the database to be ready
|
||||
DBReadyTimeout time.Duration |
||||
|
||||
// StopTimeout is how long to wait for processes to stop gracefully
|
||||
StopTimeout time.Duration |
||||
|
||||
// DataDir is the data directory to pass to orly-db
|
||||
DataDir string |
||||
|
||||
// LogLevel is the log level to use for both processes
|
||||
LogLevel string |
||||
} |
||||
|
||||
func loadConfig() (*Config, error) { |
||||
cfg := &Config{ |
||||
DBBinary: getEnvOrDefault("ORLY_LAUNCHER_DB_BINARY", "orly-db"), |
||||
RelayBinary: getEnvOrDefault("ORLY_LAUNCHER_RELAY_BINARY", "orly"), |
||||
DBListen: getEnvOrDefault("ORLY_LAUNCHER_DB_LISTEN", "127.0.0.1:50051"), |
||||
DBReadyTimeout: parseDuration("ORLY_LAUNCHER_DB_READY_TIMEOUT", 30*time.Second), |
||||
StopTimeout: parseDuration("ORLY_LAUNCHER_STOP_TIMEOUT", 30*time.Second), // Increased for DB flush
|
||||
DataDir: getEnvOrDefault("ORLY_DATA_DIR", filepath.Join(xdg.DataHome, "ORLY")), |
||||
LogLevel: getEnvOrDefault("ORLY_LOG_LEVEL", "info"), |
||||
} |
||||
|
||||
return cfg, nil |
||||
} |
||||
|
||||
func getEnvOrDefault(key, defaultValue string) string { |
||||
if v := os.Getenv(key); v != "" { |
||||
return v |
||||
} |
||||
return defaultValue |
||||
} |
||||
|
||||
func parseDuration(key string, defaultValue time.Duration) time.Duration { |
||||
if v := os.Getenv(key); v != "" { |
||||
if d, err := time.ParseDuration(v); err == nil { |
||||
return d |
||||
} |
||||
} |
||||
return defaultValue |
||||
} |
||||
@ -0,0 +1,109 @@
@@ -0,0 +1,109 @@
|
||||
// orly-launcher is a process supervisor that manages the database and relay
|
||||
// processes in split mode. It starts the database server first, waits for it
|
||||
// to be ready, then starts the relay with the gRPC database backend.
|
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"os" |
||||
"os/signal" |
||||
"syscall" |
||||
|
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
"next.orly.dev/pkg/version" |
||||
) |
||||
|
||||
func main() { |
||||
cfg, err := loadConfig() |
||||
if err != nil { |
||||
fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) |
||||
os.Exit(1) |
||||
} |
||||
|
||||
// Handle version request
|
||||
if len(os.Args) > 1 && (os.Args[1] == "version" || os.Args[1] == "-v" || os.Args[1] == "--version") { |
||||
fmt.Println(version.V) |
||||
os.Exit(0) |
||||
} |
||||
|
||||
// Handle help request
|
||||
if len(os.Args) > 1 && (os.Args[1] == "help" || os.Args[1] == "-h" || os.Args[1] == "--help") { |
||||
printHelp() |
||||
os.Exit(0) |
||||
} |
||||
|
||||
ctx, cancel := context.WithCancel(context.Background()) |
||||
defer cancel() |
||||
|
||||
supervisor := NewSupervisor(ctx, cancel, cfg) |
||||
|
||||
// Handle shutdown signals
|
||||
sigChan := make(chan os.Signal, 1) |
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) |
||||
|
||||
go func() { |
||||
sig := <-sigChan |
||||
log.I.F("received signal %v, shutting down...", sig) |
||||
cancel() |
||||
}() |
||||
|
||||
log.I.F("starting orly-launcher %s", version.V) |
||||
log.I.F("database binary: %s", cfg.DBBinary) |
||||
log.I.F("relay binary: %s", cfg.RelayBinary) |
||||
log.I.F("database listen: %s", cfg.DBListen) |
||||
|
||||
if err := supervisor.Start(); chk.E(err) { |
||||
fmt.Fprintf(os.Stderr, "failed to start: %v\n", err) |
||||
os.Exit(1) |
||||
} |
||||
|
||||
// Wait for context cancellation (signal received)
|
||||
<-ctx.Done() |
||||
|
||||
log.I.F("stopping supervisor...") |
||||
if err := supervisor.Stop(); chk.E(err) { |
||||
log.E.F("error during shutdown: %v", err) |
||||
} |
||||
|
||||
log.I.F("orly-launcher stopped") |
||||
} |
||||
|
||||
func printHelp() { |
||||
fmt.Printf(`orly-launcher %s |
||||
|
||||
Process supervisor for split-mode deployment of ORLY relay. |
||||
|
||||
Usage: orly-launcher [command] |
||||
|
||||
Commands: |
||||
help, -h, --help Show this help |
||||
version, -v, --version Show version |
||||
|
||||
Environment Variables: |
||||
ORLY_LAUNCHER_DB_BINARY Path to orly-db binary (default: orly-db) |
||||
ORLY_LAUNCHER_RELAY_BINARY Path to orly binary (default: orly) |
||||
ORLY_LAUNCHER_DB_LISTEN Address for database server (default: 127.0.0.1:50051) |
||||
ORLY_LAUNCHER_DB_READY_TIMEOUT Timeout waiting for DB ready (default: 30s) |
||||
ORLY_LAUNCHER_STOP_TIMEOUT Timeout for graceful stop (default: 10s) |
||||
ORLY_DATA_DIR Data directory (passed to orly-db) |
||||
ORLY_DB_LOG_LEVEL Database log level (passed to orly-db) |
||||
|
||||
The launcher will: |
||||
1. Start the database server (orly-db) |
||||
2. Wait for the database to be ready |
||||
3. Start the relay (orly) with ORLY_DB_TYPE=grpc |
||||
4. Monitor both processes and restart if they crash |
||||
5. On shutdown, stop relay first, then database |
||||
|
||||
Example: |
||||
# Start with default binaries in PATH |
||||
orly-launcher |
||||
|
||||
# Start with custom binary paths |
||||
ORLY_LAUNCHER_DB_BINARY=/opt/orly/orly-db \
|
||||
ORLY_LAUNCHER_RELAY_BINARY=/opt/orly/orly \
|
||||
orly-launcher |
||||
`, version.V) |
||||
} |
||||
@ -0,0 +1,310 @@
@@ -0,0 +1,310 @@
|
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"net" |
||||
"os" |
||||
"os/exec" |
||||
"sync" |
||||
"syscall" |
||||
"time" |
||||
|
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
) |
||||
|
||||
// Supervisor manages the database and relay processes.
|
||||
type Supervisor struct { |
||||
cfg *Config |
||||
ctx context.Context |
||||
cancel context.CancelFunc |
||||
|
||||
dbProc *Process |
||||
relayProc *Process |
||||
|
||||
wg sync.WaitGroup |
||||
mu sync.Mutex |
||||
closed bool |
||||
} |
||||
|
||||
// Process represents a managed subprocess.
|
||||
type Process struct { |
||||
name string |
||||
cmd *exec.Cmd |
||||
restarts int |
||||
exited chan struct{} // closed when process exits
|
||||
mu sync.Mutex |
||||
} |
||||
|
||||
// NewSupervisor creates a new process supervisor.
|
||||
func NewSupervisor(ctx context.Context, cancel context.CancelFunc, cfg *Config) *Supervisor { |
||||
return &Supervisor{ |
||||
cfg: cfg, |
||||
ctx: ctx, |
||||
cancel: cancel, |
||||
} |
||||
} |
||||
|
||||
// Start starts the database and relay processes.
|
||||
func (s *Supervisor) Start() error { |
||||
// 1. Start database server
|
||||
if err := s.startDB(); err != nil { |
||||
return fmt.Errorf("failed to start database: %w", err) |
||||
} |
||||
|
||||
// 2. Wait for DB to be ready (health check on gRPC port)
|
||||
if err := s.waitForDBReady(s.cfg.DBReadyTimeout); err != nil { |
||||
s.stopDB() |
||||
return fmt.Errorf("database not ready: %w", err) |
||||
} |
||||
|
||||
log.I.F("database is ready") |
||||
|
||||
// 3. Start relay with gRPC backend
|
||||
if err := s.startRelay(); err != nil { |
||||
s.stopDB() |
||||
return fmt.Errorf("failed to start relay: %w", err) |
||||
} |
||||
|
||||
// 4. Start monitoring goroutines
|
||||
s.wg.Add(2) |
||||
go s.monitorProcess(s.dbProc, "db", s.startDB) |
||||
go s.monitorProcess(s.relayProc, "relay", s.startRelay) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Stop stops all managed processes gracefully.
|
||||
func (s *Supervisor) Stop() error { |
||||
s.mu.Lock() |
||||
if s.closed { |
||||
s.mu.Unlock() |
||||
return nil |
||||
} |
||||
s.closed = true |
||||
s.mu.Unlock() |
||||
|
||||
// Stop relay first (it depends on DB)
|
||||
log.I.F("stopping relay...") |
||||
s.stopProcess(s.relayProc, 5*time.Second) |
||||
|
||||
// Stop DB with longer timeout for flush
|
||||
log.I.F("stopping database...") |
||||
s.stopProcess(s.dbProc, s.cfg.StopTimeout) |
||||
|
||||
// Wait for monitor goroutines to exit (they will exit when they see closed=true)
|
||||
s.wg.Wait() |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (s *Supervisor) startDB() error { |
||||
s.mu.Lock() |
||||
defer s.mu.Unlock() |
||||
|
||||
// Build environment for database process
|
||||
env := os.Environ() |
||||
env = append(env, fmt.Sprintf("ORLY_DB_LISTEN=%s", s.cfg.DBListen)) |
||||
env = append(env, fmt.Sprintf("ORLY_DATA_DIR=%s", s.cfg.DataDir)) |
||||
env = append(env, fmt.Sprintf("ORLY_DB_LOG_LEVEL=%s", s.cfg.LogLevel)) |
||||
|
||||
cmd := exec.CommandContext(s.ctx, s.cfg.DBBinary) |
||||
cmd.Env = env |
||||
cmd.Stdout = os.Stdout |
||||
cmd.Stderr = os.Stderr |
||||
|
||||
if err := cmd.Start(); chk.E(err) { |
||||
return err |
||||
} |
||||
|
||||
exited := make(chan struct{}) |
||||
s.dbProc = &Process{ |
||||
name: "orly-db", |
||||
cmd: cmd, |
||||
exited: exited, |
||||
} |
||||
|
||||
// Start a goroutine to wait for the process and close the exited channel
|
||||
go func() { |
||||
cmd.Wait() |
||||
close(exited) |
||||
}() |
||||
|
||||
log.I.F("started database server (pid %d)", cmd.Process.Pid) |
||||
return nil |
||||
} |
||||
|
||||
func (s *Supervisor) startRelay() error { |
||||
s.mu.Lock() |
||||
defer s.mu.Unlock() |
||||
|
||||
// Build environment for relay process
|
||||
env := os.Environ() |
||||
env = append(env, "ORLY_DB_TYPE=grpc") |
||||
env = append(env, fmt.Sprintf("ORLY_GRPC_SERVER=%s", s.cfg.DBListen)) |
||||
env = append(env, fmt.Sprintf("ORLY_LOG_LEVEL=%s", s.cfg.LogLevel)) |
||||
|
||||
cmd := exec.CommandContext(s.ctx, s.cfg.RelayBinary) |
||||
cmd.Env = env |
||||
cmd.Stdout = os.Stdout |
||||
cmd.Stderr = os.Stderr |
||||
|
||||
if err := cmd.Start(); chk.E(err) { |
||||
return err |
||||
} |
||||
|
||||
exited := make(chan struct{}) |
||||
s.relayProc = &Process{ |
||||
name: "orly", |
||||
cmd: cmd, |
||||
exited: exited, |
||||
} |
||||
|
||||
// Start a goroutine to wait for the process and close the exited channel
|
||||
go func() { |
||||
cmd.Wait() |
||||
close(exited) |
||||
}() |
||||
|
||||
log.I.F("started relay server (pid %d)", cmd.Process.Pid) |
||||
return nil |
||||
} |
||||
|
||||
func (s *Supervisor) waitForDBReady(timeout time.Duration) error { |
||||
deadline := time.Now().Add(timeout) |
||||
ticker := time.NewTicker(250 * time.Millisecond) |
||||
defer ticker.Stop() |
||||
|
||||
for { |
||||
select { |
||||
case <-s.ctx.Done(): |
||||
return s.ctx.Err() |
||||
case <-ticker.C: |
||||
if time.Now().After(deadline) { |
||||
return fmt.Errorf("timeout waiting for database") |
||||
} |
||||
|
||||
// Try to connect to the gRPC port
|
||||
conn, err := net.DialTimeout("tcp", s.cfg.DBListen, time.Second) |
||||
if err == nil { |
||||
conn.Close() |
||||
return nil // Database is accepting connections
|
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (s *Supervisor) stopDB() { |
||||
s.stopProcess(s.dbProc, s.cfg.StopTimeout) |
||||
} |
||||
|
||||
func (s *Supervisor) stopProcess(p *Process, timeout time.Duration) { |
||||
if p == nil { |
||||
return |
||||
} |
||||
|
||||
p.mu.Lock() |
||||
if p.cmd == nil || p.cmd.Process == nil { |
||||
p.mu.Unlock() |
||||
return |
||||
} |
||||
p.mu.Unlock() |
||||
|
||||
// Send SIGTERM for graceful shutdown
|
||||
if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil { |
||||
// Process may have already exited
|
||||
log.D.F("%s already exited: %v", p.name, err) |
||||
return |
||||
} |
||||
|
||||
// Wait for process to exit using the exited channel
|
||||
select { |
||||
case <-p.exited: |
||||
log.I.F("%s stopped gracefully", p.name) |
||||
case <-time.After(timeout): |
||||
log.W.F("%s did not stop in time, killing", p.name) |
||||
p.cmd.Process.Kill() |
||||
<-p.exited // Wait for the kill to complete
|
||||
} |
||||
} |
||||
|
||||
func (s *Supervisor) monitorProcess(p *Process, procType string, restart func() error) { |
||||
defer s.wg.Done() |
||||
|
||||
for { |
||||
// Check if we're shutting down
|
||||
s.mu.Lock() |
||||
closed := s.closed |
||||
s.mu.Unlock() |
||||
|
||||
if closed { |
||||
return |
||||
} |
||||
|
||||
select { |
||||
case <-s.ctx.Done(): |
||||
return |
||||
default: |
||||
} |
||||
|
||||
if p == nil || p.exited == nil { |
||||
return |
||||
} |
||||
|
||||
// Wait for process to exit
|
||||
select { |
||||
case <-p.exited: |
||||
// Process exited
|
||||
case <-s.ctx.Done(): |
||||
return |
||||
} |
||||
|
||||
// Check again if we're shutting down (process may have been stopped intentionally)
|
||||
s.mu.Lock() |
||||
closed = s.closed |
||||
s.mu.Unlock() |
||||
|
||||
if closed { |
||||
return |
||||
} |
||||
|
||||
// Process exited unexpectedly
|
||||
p.restarts++ |
||||
log.W.F("%s exited unexpectedly, restart count: %d", p.name, p.restarts) |
||||
|
||||
// Backoff before restart
|
||||
backoff := time.Duration(p.restarts) * time.Second |
||||
if backoff > 30*time.Second { |
||||
backoff = 30 * time.Second |
||||
} |
||||
|
||||
select { |
||||
case <-s.ctx.Done(): |
||||
return |
||||
case <-time.After(backoff): |
||||
} |
||||
|
||||
// Check one more time before restarting
|
||||
s.mu.Lock() |
||||
closed = s.closed |
||||
s.mu.Unlock() |
||||
|
||||
if closed { |
||||
return |
||||
} |
||||
|
||||
if err := restart(); err != nil { |
||||
log.E.F("failed to restart %s: %v", p.name, err) |
||||
} else { |
||||
// Update p to point to the new process
|
||||
s.mu.Lock() |
||||
if procType == "db" { |
||||
p = s.dbProc |
||||
} else { |
||||
p = s.relayProc |
||||
} |
||||
s.mu.Unlock() |
||||
} |
||||
} |
||||
} |
||||
@ -1,330 +0,0 @@
@@ -1,330 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"sync" |
||||
"time" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"lol.mleku.dev/chk" |
||||
) |
||||
|
||||
// BatchedWrite represents a single write operation
|
||||
type BatchedWrite struct { |
||||
BucketName []byte |
||||
Key []byte |
||||
Value []byte |
||||
IsDelete bool |
||||
} |
||||
|
||||
// EventBatch represents a complete event with all its indexes and graph updates
|
||||
type EventBatch struct { |
||||
Serial uint64 |
||||
EventData []byte // Serialized compact event data
|
||||
Indexes []BatchedWrite // Index entries
|
||||
EventVertex *EventVertex // Graph vertex for this event
|
||||
PubkeyUpdate *PubkeyVertexUpdate // Update to author's pubkey vertex
|
||||
MentionUpdates []*PubkeyVertexUpdate // Updates to mentioned pubkeys
|
||||
EdgeKeys []EdgeKey // Edge keys for bloom filter
|
||||
} |
||||
|
||||
// PubkeyVertexUpdate represents an update to a pubkey's vertex
|
||||
type PubkeyVertexUpdate struct { |
||||
PubkeySerial uint64 |
||||
AddAuthored uint64 // Event serial to add to authored (0 if none)
|
||||
AddMention uint64 // Event serial to add to mentions (0 if none)
|
||||
} |
||||
|
||||
// WriteBatcher accumulates writes and flushes them in batches.
|
||||
// Optimized for HDD with large batches and periodic flushes.
|
||||
type WriteBatcher struct { |
||||
db *bolt.DB |
||||
bloom *EdgeBloomFilter |
||||
logger *Logger |
||||
|
||||
mu sync.Mutex |
||||
pending []*EventBatch |
||||
pendingSize int64 |
||||
stopped bool |
||||
|
||||
// Configuration
|
||||
maxEvents int |
||||
maxBytes int64 |
||||
flushPeriod time.Duration |
||||
|
||||
// Channels for coordination
|
||||
flushCh chan struct{} |
||||
shutdownCh chan struct{} |
||||
doneCh chan struct{} |
||||
|
||||
// Stats
|
||||
stats BatcherStats |
||||
} |
||||
|
||||
// BatcherStats contains batcher statistics
|
||||
type BatcherStats struct { |
||||
TotalBatches uint64 |
||||
TotalEvents uint64 |
||||
TotalBytes uint64 |
||||
AverageLatencyMs float64 |
||||
LastFlushTime time.Time |
||||
LastFlushDuration time.Duration |
||||
} |
||||
|
||||
// NewWriteBatcher creates a new write batcher
|
||||
func NewWriteBatcher(db *bolt.DB, bloom *EdgeBloomFilter, cfg *BboltConfig, logger *Logger) *WriteBatcher { |
||||
wb := &WriteBatcher{ |
||||
db: db, |
||||
bloom: bloom, |
||||
logger: logger, |
||||
maxEvents: cfg.BatchMaxEvents, |
||||
maxBytes: cfg.BatchMaxBytes, |
||||
flushPeriod: cfg.BatchFlushTimeout, |
||||
pending: make([]*EventBatch, 0, cfg.BatchMaxEvents), |
||||
flushCh: make(chan struct{}, 1), |
||||
shutdownCh: make(chan struct{}), |
||||
doneCh: make(chan struct{}), |
||||
} |
||||
|
||||
go wb.flushLoop() |
||||
return wb |
||||
} |
||||
|
||||
// Add adds an event batch to the pending writes
|
||||
func (wb *WriteBatcher) Add(batch *EventBatch) error { |
||||
wb.mu.Lock() |
||||
defer wb.mu.Unlock() |
||||
|
||||
if wb.stopped { |
||||
return ErrBatcherStopped |
||||
} |
||||
|
||||
wb.pending = append(wb.pending, batch) |
||||
wb.pendingSize += int64(len(batch.EventData)) |
||||
for _, idx := range batch.Indexes { |
||||
wb.pendingSize += int64(len(idx.Key) + len(idx.Value)) |
||||
} |
||||
|
||||
// Check thresholds
|
||||
if len(wb.pending) >= wb.maxEvents || wb.pendingSize >= wb.maxBytes { |
||||
wb.triggerFlush() |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// triggerFlush signals the flush loop to flush (must be called with lock held)
|
||||
func (wb *WriteBatcher) triggerFlush() { |
||||
select { |
||||
case wb.flushCh <- struct{}{}: |
||||
default: |
||||
// Already a flush pending
|
||||
} |
||||
} |
||||
|
||||
// flushLoop runs the background flush timer
|
||||
func (wb *WriteBatcher) flushLoop() { |
||||
defer close(wb.doneCh) |
||||
|
||||
timer := time.NewTimer(wb.flushPeriod) |
||||
defer timer.Stop() |
||||
|
||||
for { |
||||
select { |
||||
case <-timer.C: |
||||
if err := wb.Flush(); err != nil { |
||||
wb.logger.Errorf("bbolt: flush error: %v", err) |
||||
} |
||||
timer.Reset(wb.flushPeriod) |
||||
|
||||
case <-wb.flushCh: |
||||
if err := wb.Flush(); err != nil { |
||||
wb.logger.Errorf("bbolt: flush error: %v", err) |
||||
} |
||||
if !timer.Stop() { |
||||
select { |
||||
case <-timer.C: |
||||
default: |
||||
} |
||||
} |
||||
timer.Reset(wb.flushPeriod) |
||||
|
||||
case <-wb.shutdownCh: |
||||
// Final flush
|
||||
if err := wb.Flush(); err != nil { |
||||
wb.logger.Errorf("bbolt: final flush error: %v", err) |
||||
} |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Flush writes all pending batches to BBolt
|
||||
func (wb *WriteBatcher) Flush() error { |
||||
wb.mu.Lock() |
||||
if len(wb.pending) == 0 { |
||||
wb.mu.Unlock() |
||||
return nil |
||||
} |
||||
|
||||
// Swap out pending slice
|
||||
toFlush := wb.pending |
||||
toFlushSize := wb.pendingSize |
||||
wb.pending = make([]*EventBatch, 0, wb.maxEvents) |
||||
wb.pendingSize = 0 |
||||
wb.mu.Unlock() |
||||
|
||||
startTime := time.Now() |
||||
|
||||
// Collect all edge keys for bloom filter update
|
||||
var allEdgeKeys []EdgeKey |
||||
for _, batch := range toFlush { |
||||
allEdgeKeys = append(allEdgeKeys, batch.EdgeKeys...) |
||||
} |
||||
|
||||
// Update bloom filter first (memory only)
|
||||
if len(allEdgeKeys) > 0 { |
||||
wb.bloom.AddBatch(allEdgeKeys) |
||||
} |
||||
|
||||
// Write all batches in a single transaction
|
||||
err := wb.db.Update(func(tx *bolt.Tx) error { |
||||
for _, batch := range toFlush { |
||||
// Write compact event data
|
||||
cmpBucket := tx.Bucket(bucketCmp) |
||||
if cmpBucket != nil { |
||||
key := makeSerialKey(batch.Serial) |
||||
if err := cmpBucket.Put(key, batch.EventData); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
// Write all indexes
|
||||
for _, idx := range batch.Indexes { |
||||
bucket := tx.Bucket(idx.BucketName) |
||||
if bucket == nil { |
||||
continue |
||||
} |
||||
if idx.IsDelete { |
||||
if err := bucket.Delete(idx.Key); err != nil { |
||||
return err |
||||
} |
||||
} else { |
||||
if err := bucket.Put(idx.Key, idx.Value); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Write event vertex
|
||||
if batch.EventVertex != nil { |
||||
evBucket := tx.Bucket(bucketEv) |
||||
if evBucket != nil { |
||||
key := makeSerialKey(batch.Serial) |
||||
if err := evBucket.Put(key, batch.EventVertex.Encode()); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Update pubkey vertices
|
||||
if err := wb.updatePubkeyVertex(tx, batch.PubkeyUpdate); err != nil { |
||||
return err |
||||
} |
||||
for _, mentionUpdate := range batch.MentionUpdates { |
||||
if err := wb.updatePubkeyVertex(tx, mentionUpdate); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
// Update stats
|
||||
duration := time.Since(startTime) |
||||
wb.mu.Lock() |
||||
wb.stats.TotalBatches++ |
||||
wb.stats.TotalEvents += uint64(len(toFlush)) |
||||
wb.stats.TotalBytes += uint64(toFlushSize) |
||||
wb.stats.LastFlushTime = time.Now() |
||||
wb.stats.LastFlushDuration = duration |
||||
latencyMs := float64(duration.Milliseconds()) |
||||
wb.stats.AverageLatencyMs = (wb.stats.AverageLatencyMs*float64(wb.stats.TotalBatches-1) + latencyMs) / float64(wb.stats.TotalBatches) |
||||
wb.mu.Unlock() |
||||
|
||||
if err == nil { |
||||
wb.logger.Debugf("bbolt: flushed %d events (%d bytes) in %v", len(toFlush), toFlushSize, duration) |
||||
} |
||||
|
||||
return err |
||||
} |
||||
|
||||
// updatePubkeyVertex reads, updates, and writes a pubkey vertex
|
||||
func (wb *WriteBatcher) updatePubkeyVertex(tx *bolt.Tx, update *PubkeyVertexUpdate) error { |
||||
if update == nil { |
||||
return nil |
||||
} |
||||
|
||||
pvBucket := tx.Bucket(bucketPv) |
||||
if pvBucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
key := makeSerialKey(update.PubkeySerial) |
||||
|
||||
// Load existing vertex or create new
|
||||
var pv PubkeyVertex |
||||
existing := pvBucket.Get(key) |
||||
if existing != nil { |
||||
if err := pv.Decode(existing); chk.E(err) { |
||||
// If decode fails, start fresh
|
||||
pv = PubkeyVertex{} |
||||
} |
||||
} |
||||
|
||||
// Apply updates
|
||||
if update.AddAuthored != 0 { |
||||
pv.AddAuthored(update.AddAuthored) |
||||
} |
||||
if update.AddMention != 0 { |
||||
pv.AddMention(update.AddMention) |
||||
} |
||||
|
||||
// Write back
|
||||
return pvBucket.Put(key, pv.Encode()) |
||||
} |
||||
|
||||
// Shutdown gracefully shuts down the batcher
|
||||
func (wb *WriteBatcher) Shutdown() error { |
||||
wb.mu.Lock() |
||||
wb.stopped = true |
||||
wb.mu.Unlock() |
||||
|
||||
close(wb.shutdownCh) |
||||
<-wb.doneCh |
||||
return nil |
||||
} |
||||
|
||||
// Stats returns current batcher statistics
|
||||
func (wb *WriteBatcher) Stats() BatcherStats { |
||||
wb.mu.Lock() |
||||
defer wb.mu.Unlock() |
||||
return wb.stats |
||||
} |
||||
|
||||
// PendingCount returns the number of pending events
|
||||
func (wb *WriteBatcher) PendingCount() int { |
||||
wb.mu.Lock() |
||||
defer wb.mu.Unlock() |
||||
return len(wb.pending) |
||||
} |
||||
|
||||
// ErrBatcherStopped is returned when adding to a stopped batcher
|
||||
var ErrBatcherStopped = &batcherStoppedError{} |
||||
|
||||
type batcherStoppedError struct{} |
||||
|
||||
func (e *batcherStoppedError) Error() string { |
||||
return "batcher has been stopped" |
||||
} |
||||
@ -1,325 +0,0 @@
@@ -1,325 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"os" |
||||
"path/filepath" |
||||
"sync" |
||||
"time" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"lol.mleku.dev" |
||||
"lol.mleku.dev/chk" |
||||
"next.orly.dev/pkg/database" |
||||
"next.orly.dev/pkg/utils/apputil" |
||||
) |
||||
|
||||
// Bucket names - map to existing index prefixes but without the 3-byte prefix in keys
|
||||
var ( |
||||
bucketEvt = []byte("evt") // Event storage: serial -> compact event data
|
||||
bucketEid = []byte("eid") // Event ID index
|
||||
bucketFpc = []byte("fpc") // Full ID/pubkey index
|
||||
bucketC = []byte("c--") // Created at index
|
||||
bucketKc = []byte("kc-") // Kind + created index
|
||||
bucketPc = []byte("pc-") // Pubkey + created index
|
||||
bucketKpc = []byte("kpc") // Kind + pubkey + created
|
||||
bucketTc = []byte("tc-") // Tag + created
|
||||
bucketTkc = []byte("tkc") // Tag + kind + created
|
||||
bucketTpc = []byte("tpc") // Tag + pubkey + created
|
||||
bucketTkp = []byte("tkp") // Tag + kind + pubkey + created
|
||||
bucketWrd = []byte("wrd") // Word search index
|
||||
bucketExp = []byte("exp") // Expiration index
|
||||
bucketPks = []byte("pks") // Pubkey hash -> serial
|
||||
bucketSpk = []byte("spk") // Serial -> pubkey
|
||||
bucketSei = []byte("sei") // Serial -> event ID
|
||||
bucketCmp = []byte("cmp") // Compact event storage
|
||||
bucketEv = []byte("ev") // Event vertices (adjacency list)
|
||||
bucketPv = []byte("pv") // Pubkey vertices (adjacency list)
|
||||
bucketMeta = []byte("_meta") // Markers, version, serial counter, bloom filter
|
||||
) |
||||
|
||||
// All buckets that need to be created on init
|
||||
var allBuckets = [][]byte{ |
||||
bucketEvt, bucketEid, bucketFpc, bucketC, bucketKc, bucketPc, bucketKpc, |
||||
bucketTc, bucketTkc, bucketTpc, bucketTkp, bucketWrd, bucketExp, |
||||
bucketPks, bucketSpk, bucketSei, bucketCmp, bucketEv, bucketPv, bucketMeta, |
||||
} |
||||
|
||||
// B implements the database.Database interface using BBolt as the storage backend.
|
||||
// Optimized for HDD with write batching and adjacency list graph storage.
|
||||
type B struct { |
||||
ctx context.Context |
||||
cancel context.CancelFunc |
||||
dataDir string |
||||
Logger *Logger |
||||
|
||||
db *bolt.DB |
||||
ready chan struct{} |
||||
|
||||
// Write batching
|
||||
batcher *WriteBatcher |
||||
|
||||
// Serial management
|
||||
serialMu sync.Mutex |
||||
nextSerial uint64 |
||||
nextPubkeySeq uint64 |
||||
|
||||
// Edge bloom filter for fast negative lookups
|
||||
edgeBloom *EdgeBloomFilter |
||||
|
||||
// Configuration
|
||||
cfg *BboltConfig |
||||
} |
||||
|
||||
// BboltConfig holds bbolt-specific configuration
|
||||
type BboltConfig struct { |
||||
DataDir string |
||||
LogLevel string |
||||
|
||||
// Batch settings (tuned for 7200rpm HDD)
|
||||
BatchMaxEvents int // Max events before flush (default: 5000)
|
||||
BatchMaxBytes int64 // Max bytes before flush (default: 128MB)
|
||||
BatchFlushTimeout time.Duration // Max time before flush (default: 30s)
|
||||
|
||||
// Bloom filter settings
|
||||
BloomSizeMB int // Bloom filter size in MB (default: 16)
|
||||
|
||||
// BBolt settings
|
||||
NoSync bool // Disable fsync for performance (DANGEROUS)
|
||||
InitialMmapSize int // Initial mmap size in bytes
|
||||
} |
||||
|
||||
// Ensure B implements Database interface at compile time
|
||||
var _ database.Database = (*B)(nil) |
||||
|
||||
// New creates a new BBolt database instance with default configuration.
|
||||
func New( |
||||
ctx context.Context, cancel context.CancelFunc, dataDir, logLevel string, |
||||
) (b *B, err error) { |
||||
cfg := &BboltConfig{ |
||||
DataDir: dataDir, |
||||
LogLevel: logLevel, |
||||
BatchMaxEvents: 5000, |
||||
BatchMaxBytes: 128 * 1024 * 1024, // 128MB
|
||||
BatchFlushTimeout: 30 * time.Second, |
||||
BloomSizeMB: 16, |
||||
InitialMmapSize: 8 * 1024 * 1024 * 1024, // 8GB
|
||||
} |
||||
return NewWithConfig(ctx, cancel, cfg) |
||||
} |
||||
|
||||
// NewWithConfig creates a new BBolt database instance with full configuration.
|
||||
func NewWithConfig( |
||||
ctx context.Context, cancel context.CancelFunc, cfg *BboltConfig, |
||||
) (b *B, err error) { |
||||
// Apply defaults
|
||||
if cfg.BatchMaxEvents <= 0 { |
||||
cfg.BatchMaxEvents = 5000 |
||||
} |
||||
if cfg.BatchMaxBytes <= 0 { |
||||
cfg.BatchMaxBytes = 128 * 1024 * 1024 |
||||
} |
||||
if cfg.BatchFlushTimeout <= 0 { |
||||
cfg.BatchFlushTimeout = 30 * time.Second |
||||
} |
||||
if cfg.BloomSizeMB <= 0 { |
||||
cfg.BloomSizeMB = 16 |
||||
} |
||||
if cfg.InitialMmapSize <= 0 { |
||||
cfg.InitialMmapSize = 8 * 1024 * 1024 * 1024 |
||||
} |
||||
|
||||
b = &B{ |
||||
ctx: ctx, |
||||
cancel: cancel, |
||||
dataDir: cfg.DataDir, |
||||
Logger: NewLogger(lol.GetLogLevel(cfg.LogLevel), cfg.DataDir), |
||||
ready: make(chan struct{}), |
||||
cfg: cfg, |
||||
} |
||||
|
||||
// Ensure the data directory exists
|
||||
if err = os.MkdirAll(cfg.DataDir, 0755); chk.E(err) { |
||||
return |
||||
} |
||||
if err = apputil.EnsureDir(filepath.Join(cfg.DataDir, "dummy")); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Open BBolt database
|
||||
dbPath := filepath.Join(cfg.DataDir, "orly.db") |
||||
opts := &bolt.Options{ |
||||
Timeout: 10 * time.Second, |
||||
NoSync: cfg.NoSync, |
||||
InitialMmapSize: cfg.InitialMmapSize, |
||||
} |
||||
|
||||
if b.db, err = bolt.Open(dbPath, 0600, opts); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Create all buckets
|
||||
if err = b.db.Update(func(tx *bolt.Tx) error { |
||||
for _, bucket := range allBuckets { |
||||
if _, err := tx.CreateBucketIfNotExists(bucket); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Initialize serial counters
|
||||
if err = b.initSerialCounters(); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Initialize bloom filter
|
||||
b.edgeBloom, err = NewEdgeBloomFilter(cfg.BloomSizeMB, b.db) |
||||
if chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Initialize write batcher
|
||||
b.batcher = NewWriteBatcher(b.db, b.edgeBloom, cfg, b.Logger) |
||||
|
||||
// Run migrations
|
||||
b.RunMigrations() |
||||
|
||||
// Start warmup and mark ready
|
||||
go b.warmup() |
||||
|
||||
// Start background maintenance
|
||||
go b.backgroundLoop() |
||||
|
||||
return |
||||
} |
||||
|
||||
// Path returns the path where the database files are stored.
|
||||
func (b *B) Path() string { return b.dataDir } |
||||
|
||||
// Init initializes the database with the given path.
|
||||
func (b *B) Init(path string) error { |
||||
b.dataDir = path |
||||
return nil |
||||
} |
||||
|
||||
// Sync flushes the database buffers to disk.
|
||||
func (b *B) Sync() error { |
||||
// Flush pending writes
|
||||
if err := b.batcher.Flush(); err != nil { |
||||
return err |
||||
} |
||||
// Persist bloom filter
|
||||
if err := b.edgeBloom.Persist(b.db); err != nil { |
||||
return err |
||||
} |
||||
// Persist serial counters
|
||||
if err := b.persistSerialCounters(); err != nil { |
||||
return err |
||||
} |
||||
// Sync BBolt
|
||||
return b.db.Sync() |
||||
} |
||||
|
||||
// Close releases resources and closes the database.
|
||||
func (b *B) Close() (err error) { |
||||
b.Logger.Infof("bbolt: closing database...") |
||||
|
||||
// Stop accepting new writes and flush pending
|
||||
if b.batcher != nil { |
||||
if err = b.batcher.Shutdown(); chk.E(err) { |
||||
// Log but continue cleanup
|
||||
} |
||||
} |
||||
|
||||
// Persist bloom filter
|
||||
if b.edgeBloom != nil { |
||||
if err = b.edgeBloom.Persist(b.db); chk.E(err) { |
||||
// Log but continue cleanup
|
||||
} |
||||
} |
||||
|
||||
// Persist serial counters
|
||||
if err = b.persistSerialCounters(); chk.E(err) { |
||||
// Log but continue cleanup
|
||||
} |
||||
|
||||
// Close BBolt database
|
||||
if b.db != nil { |
||||
if err = b.db.Close(); chk.E(err) { |
||||
return |
||||
} |
||||
} |
||||
|
||||
b.Logger.Infof("bbolt: database closed") |
||||
return |
||||
} |
||||
|
||||
// Wipe deletes all data in the database.
|
||||
func (b *B) Wipe() error { |
||||
return b.db.Update(func(tx *bolt.Tx) error { |
||||
for _, bucket := range allBuckets { |
||||
if err := tx.DeleteBucket(bucket); err != nil && !errors.Is(err, bolt.ErrBucketNotFound) { |
||||
return err |
||||
} |
||||
if _, err := tx.CreateBucket(bucket); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
// Reset serial counters
|
||||
b.serialMu.Lock() |
||||
b.nextSerial = 1 |
||||
b.nextPubkeySeq = 1 |
||||
b.serialMu.Unlock() |
||||
// Reset bloom filter
|
||||
b.edgeBloom.Reset() |
||||
return nil |
||||
}) |
||||
} |
||||
|
||||
// SetLogLevel changes the logging level.
|
||||
func (b *B) SetLogLevel(level string) { |
||||
b.Logger.SetLogLevel(lol.GetLogLevel(level)) |
||||
} |
||||
|
||||
// Ready returns a channel that closes when the database is ready to serve requests.
|
||||
func (b *B) Ready() <-chan struct{} { |
||||
return b.ready |
||||
} |
||||
|
||||
// warmup performs database warmup operations and closes the ready channel when complete.
|
||||
func (b *B) warmup() { |
||||
defer close(b.ready) |
||||
|
||||
// Give the database time to settle
|
||||
time.Sleep(1 * time.Second) |
||||
|
||||
b.Logger.Infof("bbolt: database warmup complete, ready to serve requests") |
||||
} |
||||
|
||||
// backgroundLoop runs periodic maintenance tasks.
|
||||
func (b *B) backgroundLoop() { |
||||
expirationTicker := time.NewTicker(10 * time.Minute) |
||||
bloomPersistTicker := time.NewTicker(5 * time.Minute) |
||||
defer expirationTicker.Stop() |
||||
defer bloomPersistTicker.Stop() |
||||
|
||||
for { |
||||
select { |
||||
case <-expirationTicker.C: |
||||
b.DeleteExpired() |
||||
case <-bloomPersistTicker.C: |
||||
if err := b.edgeBloom.Persist(b.db); chk.E(err) { |
||||
b.Logger.Warningf("bbolt: failed to persist bloom filter: %v", err) |
||||
} |
||||
case <-b.ctx.Done(): |
||||
b.cancel() |
||||
return |
||||
} |
||||
} |
||||
} |
||||
@ -1,192 +0,0 @@
@@ -1,192 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/binary" |
||||
"sync" |
||||
|
||||
"github.com/bits-and-blooms/bloom/v3" |
||||
bolt "go.etcd.io/bbolt" |
||||
"lol.mleku.dev/chk" |
||||
) |
||||
|
||||
const bloomFilterKey = "edge_bloom_filter" |
||||
|
||||
// EdgeBloomFilter provides fast negative lookups for edge existence checks.
|
||||
// Uses a bloom filter to avoid disk seeks when checking if an edge exists.
|
||||
type EdgeBloomFilter struct { |
||||
mu sync.RWMutex |
||||
filter *bloom.BloomFilter |
||||
|
||||
// Track if filter has been modified since last persist
|
||||
dirty bool |
||||
} |
||||
|
||||
// NewEdgeBloomFilter creates or loads the edge bloom filter.
|
||||
// sizeMB is the approximate size in megabytes.
|
||||
// With 1% false positive rate, 16MB can hold ~10 million edges.
|
||||
func NewEdgeBloomFilter(sizeMB int, db *bolt.DB) (*EdgeBloomFilter, error) { |
||||
ebf := &EdgeBloomFilter{} |
||||
|
||||
// Try to load from database
|
||||
var loaded bool |
||||
err := db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
data := bucket.Get([]byte(bloomFilterKey)) |
||||
if data == nil { |
||||
return nil |
||||
} |
||||
|
||||
// Deserialize bloom filter
|
||||
reader := bytes.NewReader(data) |
||||
filter := &bloom.BloomFilter{} |
||||
if _, err := filter.ReadFrom(reader); err != nil { |
||||
return err |
||||
} |
||||
|
||||
ebf.filter = filter |
||||
loaded = true |
||||
return nil |
||||
}) |
||||
if chk.E(err) { |
||||
return nil, err |
||||
} |
||||
|
||||
if !loaded { |
||||
// Create new filter
|
||||
// Calculate parameters: m bits, k hash functions
|
||||
// For 1% false positive rate: m/n ≈ 9.6, k ≈ 7
|
||||
bitsPerMB := 8 * 1024 * 1024 |
||||
totalBits := uint(sizeMB * bitsPerMB) |
||||
// Estimate capacity based on 10 bits per element for 1% FPR
|
||||
estimatedCapacity := uint(totalBits / 10) |
||||
|
||||
ebf.filter = bloom.NewWithEstimates(estimatedCapacity, 0.01) |
||||
} |
||||
|
||||
return ebf, nil |
||||
} |
||||
|
||||
// Add adds an edge to the bloom filter.
|
||||
// An edge is represented by source and destination serials plus edge type.
|
||||
func (ebf *EdgeBloomFilter) Add(srcSerial, dstSerial uint64, edgeType byte) { |
||||
ebf.mu.Lock() |
||||
defer ebf.mu.Unlock() |
||||
|
||||
key := ebf.makeKey(srcSerial, dstSerial, edgeType) |
||||
ebf.filter.Add(key) |
||||
ebf.dirty = true |
||||
} |
||||
|
||||
// AddBatch adds multiple edges to the bloom filter.
|
||||
func (ebf *EdgeBloomFilter) AddBatch(edges []EdgeKey) { |
||||
ebf.mu.Lock() |
||||
defer ebf.mu.Unlock() |
||||
|
||||
for _, edge := range edges { |
||||
key := ebf.makeKey(edge.SrcSerial, edge.DstSerial, edge.EdgeType) |
||||
ebf.filter.Add(key) |
||||
} |
||||
ebf.dirty = true |
||||
} |
||||
|
||||
// MayExist checks if an edge might exist.
|
||||
// Returns false if definitely doesn't exist (no disk access needed).
|
||||
// Returns true if might exist (need to check disk to confirm).
|
||||
func (ebf *EdgeBloomFilter) MayExist(srcSerial, dstSerial uint64, edgeType byte) bool { |
||||
ebf.mu.RLock() |
||||
defer ebf.mu.RUnlock() |
||||
|
||||
key := ebf.makeKey(srcSerial, dstSerial, edgeType) |
||||
return ebf.filter.Test(key) |
||||
} |
||||
|
||||
// Persist saves the bloom filter to the database.
|
||||
func (ebf *EdgeBloomFilter) Persist(db *bolt.DB) error { |
||||
ebf.mu.Lock() |
||||
if !ebf.dirty { |
||||
ebf.mu.Unlock() |
||||
return nil |
||||
} |
||||
|
||||
// Serialize while holding lock
|
||||
var buf bytes.Buffer |
||||
if _, err := ebf.filter.WriteTo(&buf); err != nil { |
||||
ebf.mu.Unlock() |
||||
return err |
||||
} |
||||
data := buf.Bytes() |
||||
ebf.dirty = false |
||||
ebf.mu.Unlock() |
||||
|
||||
// Write to database
|
||||
return db.Update(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
return bucket.Put([]byte(bloomFilterKey), data) |
||||
}) |
||||
} |
||||
|
||||
// Reset clears the bloom filter.
|
||||
func (ebf *EdgeBloomFilter) Reset() { |
||||
ebf.mu.Lock() |
||||
defer ebf.mu.Unlock() |
||||
|
||||
ebf.filter.ClearAll() |
||||
ebf.dirty = true |
||||
} |
||||
|
||||
// makeKey creates a unique key for an edge.
|
||||
func (ebf *EdgeBloomFilter) makeKey(srcSerial, dstSerial uint64, edgeType byte) []byte { |
||||
key := make([]byte, 17) // 8 + 8 + 1
|
||||
binary.BigEndian.PutUint64(key[0:8], srcSerial) |
||||
binary.BigEndian.PutUint64(key[8:16], dstSerial) |
||||
key[16] = edgeType |
||||
return key |
||||
} |
||||
|
||||
// Stats returns bloom filter statistics.
|
||||
func (ebf *EdgeBloomFilter) Stats() BloomStats { |
||||
ebf.mu.RLock() |
||||
defer ebf.mu.RUnlock() |
||||
|
||||
approxCount := uint64(ebf.filter.ApproximatedSize()) |
||||
cap := ebf.filter.Cap() |
||||
|
||||
return BloomStats{ |
||||
ApproxCount: approxCount, |
||||
Cap: cap, |
||||
} |
||||
} |
||||
|
||||
// BloomStats contains bloom filter statistics.
|
||||
type BloomStats struct { |
||||
ApproxCount uint64 // Approximate number of elements
|
||||
Cap uint // Capacity in bits
|
||||
} |
||||
|
||||
// EdgeKey represents an edge for batch operations.
|
||||
type EdgeKey struct { |
||||
SrcSerial uint64 |
||||
DstSerial uint64 |
||||
EdgeType byte |
||||
} |
||||
|
||||
// Edge type constants
|
||||
const ( |
||||
EdgeTypeAuthor byte = 0 // Event author relationship
|
||||
EdgeTypePTag byte = 1 // P-tag reference (event mentions pubkey)
|
||||
EdgeTypeETag byte = 2 // E-tag reference (event references event)
|
||||
EdgeTypeFollows byte = 3 // Kind 3 follows relationship
|
||||
EdgeTypeReaction byte = 4 // Kind 7 reaction
|
||||
EdgeTypeRepost byte = 5 // Kind 6 repost
|
||||
EdgeTypeReply byte = 6 // Reply (kind 1 with e-tag)
|
||||
) |
||||
@ -1,134 +0,0 @@
@@ -1,134 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"errors" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"lol.mleku.dev/chk" |
||||
"next.orly.dev/pkg/database" |
||||
"next.orly.dev/pkg/database/indexes/types" |
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
"git.mleku.dev/mleku/nostr/encoders/filter" |
||||
) |
||||
|
||||
// FetchEventBySerial fetches an event by its serial number.
|
||||
func (b *B) FetchEventBySerial(ser *types.Uint40) (ev *event.E, err error) { |
||||
if ser == nil { |
||||
return nil, errors.New("bbolt: nil serial") |
||||
} |
||||
|
||||
serial := ser.Get() |
||||
key := makeSerialKey(serial) |
||||
|
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
// Get event ID first
|
||||
seiBucket := tx.Bucket(bucketSei) |
||||
var eventId []byte |
||||
if seiBucket != nil { |
||||
eventId = seiBucket.Get(key) |
||||
} |
||||
|
||||
// Try compact event storage first
|
||||
cmpBucket := tx.Bucket(bucketCmp) |
||||
if cmpBucket != nil { |
||||
data := cmpBucket.Get(key) |
||||
if data != nil && eventId != nil && len(eventId) == 32 { |
||||
// Unmarshal compact event
|
||||
resolver := &bboltSerialResolver{b: b} |
||||
ev, err = database.UnmarshalCompactEvent(data, eventId, resolver) |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
// Fall through to try legacy format
|
||||
} |
||||
} |
||||
|
||||
// Try legacy event storage
|
||||
evtBucket := tx.Bucket(bucketEvt) |
||||
if evtBucket != nil { |
||||
data := evtBucket.Get(key) |
||||
if data != nil { |
||||
ev = new(event.E) |
||||
reader := bytes.NewReader(data) |
||||
if err = ev.UnmarshalBinary(reader); err == nil { |
||||
return nil |
||||
} |
||||
} |
||||
} |
||||
|
||||
return errors.New("bbolt: event not found") |
||||
}) |
||||
|
||||
return |
||||
} |
||||
|
||||
// FetchEventsBySerials fetches multiple events by their serial numbers.
|
||||
func (b *B) FetchEventsBySerials(serials []*types.Uint40) (events map[uint64]*event.E, err error) { |
||||
events = make(map[uint64]*event.E, len(serials)) |
||||
|
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
cmpBucket := tx.Bucket(bucketCmp) |
||||
evtBucket := tx.Bucket(bucketEvt) |
||||
seiBucket := tx.Bucket(bucketSei) |
||||
resolver := &bboltSerialResolver{b: b} |
||||
|
||||
for _, ser := range serials { |
||||
if ser == nil { |
||||
continue |
||||
} |
||||
|
||||
serial := ser.Get() |
||||
key := makeSerialKey(serial) |
||||
|
||||
// Get event ID
|
||||
var eventId []byte |
||||
if seiBucket != nil { |
||||
eventId = seiBucket.Get(key) |
||||
} |
||||
|
||||
// Try compact event storage first
|
||||
if cmpBucket != nil { |
||||
data := cmpBucket.Get(key) |
||||
if data != nil && eventId != nil && len(eventId) == 32 { |
||||
ev, e := database.UnmarshalCompactEvent(data, eventId, resolver) |
||||
if e == nil { |
||||
events[serial] = ev |
||||
continue |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Try legacy event storage
|
||||
if evtBucket != nil { |
||||
data := evtBucket.Get(key) |
||||
if data != nil { |
||||
ev := new(event.E) |
||||
reader := bytes.NewReader(data) |
||||
if e := ev.UnmarshalBinary(reader); e == nil { |
||||
events[serial] = ev |
||||
} |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
return |
||||
} |
||||
|
||||
// CountEvents counts events matching a filter.
|
||||
func (b *B) CountEvents(c context.Context, f *filter.F) (count int, approximate bool, err error) { |
||||
// Get serials matching filter
|
||||
var serials types.Uint40s |
||||
if serials, err = b.GetSerialsFromFilter(f); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
count = len(serials) |
||||
approximate = false |
||||
return |
||||
} |
||||
@ -1,179 +0,0 @@
@@ -1,179 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"bytes" |
||||
"errors" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"next.orly.dev/pkg/database" |
||||
"next.orly.dev/pkg/database/indexes/types" |
||||
"next.orly.dev/pkg/interfaces/store" |
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
"git.mleku.dev/mleku/nostr/encoders/tag" |
||||
) |
||||
|
||||
// GetSerialById gets the serial for an event ID.
|
||||
func (b *B) GetSerialById(id []byte) (ser *types.Uint40, err error) { |
||||
if len(id) < 8 { |
||||
return nil, errors.New("bbolt: invalid event ID length") |
||||
} |
||||
|
||||
idHash := hashEventId(id) |
||||
|
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketEid) |
||||
if bucket == nil { |
||||
return errors.New("id not found in database") |
||||
} |
||||
|
||||
// Scan for matching ID hash prefix
|
||||
c := bucket.Cursor() |
||||
for k, _ := c.Seek(idHash); k != nil && bytes.HasPrefix(k, idHash); k, _ = c.Next() { |
||||
// Key format: id_hash(8) | serial(5)
|
||||
if len(k) >= 13 { |
||||
ser = new(types.Uint40) |
||||
ser.Set(decodeUint40(k[8:13])) |
||||
return nil |
||||
} |
||||
} |
||||
return errors.New("id not found in database") |
||||
}) |
||||
|
||||
return |
||||
} |
||||
|
||||
// GetSerialsByIds gets serials for multiple event IDs.
|
||||
func (b *B) GetSerialsByIds(ids *tag.T) (serials map[string]*types.Uint40, err error) { |
||||
serials = make(map[string]*types.Uint40, ids.Len()) |
||||
|
||||
if ids == nil || ids.Len() == 0 { |
||||
return |
||||
} |
||||
|
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketEid) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
// Iterate over the tag entries using the .T field
|
||||
for _, id := range ids.T { |
||||
if len(id) < 8 { |
||||
continue |
||||
} |
||||
|
||||
idHash := hashEventId(id) |
||||
c := bucket.Cursor() |
||||
for k, _ := c.Seek(idHash); k != nil && bytes.HasPrefix(k, idHash); k, _ = c.Next() { |
||||
if len(k) >= 13 { |
||||
ser := new(types.Uint40) |
||||
ser.Set(decodeUint40(k[8:13])) |
||||
serials[string(id)] = ser |
||||
break |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
return |
||||
} |
||||
|
||||
// GetSerialsByIdsWithFilter gets serials with a filter function.
|
||||
func (b *B) GetSerialsByIdsWithFilter(ids *tag.T, fn func(ev *event.E, ser *types.Uint40) bool) (serials map[string]*types.Uint40, err error) { |
||||
// For now, just call GetSerialsByIds - full implementation would apply filter
|
||||
return b.GetSerialsByIds(ids) |
||||
} |
||||
|
||||
// GetSerialsByRange gets serials within a key range.
|
||||
func (b *B) GetSerialsByRange(idx database.Range) (serials types.Uint40s, err error) { |
||||
if len(idx.Start) < 3 { |
||||
return nil, errors.New("bbolt: invalid range start") |
||||
} |
||||
|
||||
// Extract bucket name from prefix
|
||||
bucketName := idx.Start[:3] |
||||
startKey := idx.Start[3:] |
||||
endKey := idx.End[3:] |
||||
|
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketName) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
c := bucket.Cursor() |
||||
for k, _ := c.Seek(startKey); k != nil; k, _ = c.Next() { |
||||
// Check if we've passed the end
|
||||
if len(endKey) > 0 && bytes.Compare(k, endKey) >= 0 { |
||||
break |
||||
} |
||||
|
||||
// Extract serial from end of key (last 5 bytes)
|
||||
if len(k) >= 5 { |
||||
ser := new(types.Uint40) |
||||
ser.Set(decodeUint40(k[len(k)-5:])) |
||||
serials = append(serials, ser) |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
return |
||||
} |
||||
|
||||
// GetFullIdPubkeyBySerial gets full event ID and pubkey by serial.
|
||||
func (b *B) GetFullIdPubkeyBySerial(ser *types.Uint40) (fidpk *store.IdPkTs, err error) { |
||||
if ser == nil { |
||||
return nil, errors.New("bbolt: nil serial") |
||||
} |
||||
|
||||
serial := ser.Get() |
||||
key := makeSerialKey(serial) |
||||
|
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
// Get full ID/pubkey from fpc bucket
|
||||
fpcBucket := tx.Bucket(bucketFpc) |
||||
if fpcBucket == nil { |
||||
return errors.New("bbolt: fpc bucket not found") |
||||
} |
||||
|
||||
// Scan for matching serial prefix
|
||||
c := fpcBucket.Cursor() |
||||
for k, _ := c.Seek(key); k != nil && bytes.HasPrefix(k, key); k, _ = c.Next() { |
||||
// Key format: serial(5) | id(32) | pubkey_hash(8) | created_at(8)
|
||||
if len(k) >= 53 { |
||||
fidpk = &store.IdPkTs{ |
||||
Ser: serial, |
||||
} |
||||
fidpk.Id = make([]byte, 32) |
||||
copy(fidpk.Id, k[5:37]) |
||||
// Pubkey is only hash here, need to look up full pubkey
|
||||
// For now return what we have
|
||||
fidpk.Pub = make([]byte, 8) |
||||
copy(fidpk.Pub, k[37:45]) |
||||
fidpk.Ts = int64(decodeUint64(k[45:53])) |
||||
return nil |
||||
} |
||||
} |
||||
return errors.New("bbolt: serial not found in fpc index") |
||||
}) |
||||
|
||||
return |
||||
} |
||||
|
||||
// GetFullIdPubkeyBySerials gets full event IDs and pubkeys for multiple serials.
|
||||
func (b *B) GetFullIdPubkeyBySerials(sers []*types.Uint40) (fidpks []*store.IdPkTs, err error) { |
||||
fidpks = make([]*store.IdPkTs, 0, len(sers)) |
||||
|
||||
for _, ser := range sers { |
||||
fidpk, e := b.GetFullIdPubkeyBySerial(ser) |
||||
if e == nil && fidpk != nil { |
||||
fidpks = append(fidpks, fidpk) |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
@ -1,250 +0,0 @@
@@ -1,250 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/binary" |
||||
"io" |
||||
) |
||||
|
||||
// EventVertex stores the adjacency list for an event.
|
||||
// Contains the author and all edges to other events/pubkeys.
|
||||
type EventVertex struct { |
||||
AuthorSerial uint64 // Serial of the author pubkey
|
||||
Kind uint16 // Event kind
|
||||
PTagSerials []uint64 // Serials of pubkeys mentioned (p-tags)
|
||||
ETagSerials []uint64 // Serials of events referenced (e-tags)
|
||||
} |
||||
|
||||
// Encode serializes the EventVertex to bytes.
|
||||
// Format: author(5) | kind(2) | ptag_count(varint) | [ptag_serials(5)...] | etag_count(varint) | [etag_serials(5)...]
|
||||
func (ev *EventVertex) Encode() []byte { |
||||
// Calculate size
|
||||
size := 5 + 2 + 2 + len(ev.PTagSerials)*5 + 2 + len(ev.ETagSerials)*5 |
||||
buf := make([]byte, 0, size) |
||||
|
||||
// Author serial (5 bytes)
|
||||
authorBuf := make([]byte, 5) |
||||
encodeUint40(ev.AuthorSerial, authorBuf) |
||||
buf = append(buf, authorBuf...) |
||||
|
||||
// Kind (2 bytes)
|
||||
kindBuf := make([]byte, 2) |
||||
binary.BigEndian.PutUint16(kindBuf, ev.Kind) |
||||
buf = append(buf, kindBuf...) |
||||
|
||||
// P-tag count and serials
|
||||
ptagCountBuf := make([]byte, 2) |
||||
binary.BigEndian.PutUint16(ptagCountBuf, uint16(len(ev.PTagSerials))) |
||||
buf = append(buf, ptagCountBuf...) |
||||
for _, serial := range ev.PTagSerials { |
||||
serialBuf := make([]byte, 5) |
||||
encodeUint40(serial, serialBuf) |
||||
buf = append(buf, serialBuf...) |
||||
} |
||||
|
||||
// E-tag count and serials
|
||||
etagCountBuf := make([]byte, 2) |
||||
binary.BigEndian.PutUint16(etagCountBuf, uint16(len(ev.ETagSerials))) |
||||
buf = append(buf, etagCountBuf...) |
||||
for _, serial := range ev.ETagSerials { |
||||
serialBuf := make([]byte, 5) |
||||
encodeUint40(serial, serialBuf) |
||||
buf = append(buf, serialBuf...) |
||||
} |
||||
|
||||
return buf |
||||
} |
||||
|
||||
// Decode deserializes bytes into an EventVertex.
|
||||
func (ev *EventVertex) Decode(data []byte) error { |
||||
if len(data) < 9 { // minimum: author(5) + kind(2) + ptag_count(2)
|
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
|
||||
reader := bytes.NewReader(data) |
||||
|
||||
// Author serial
|
||||
authorBuf := make([]byte, 5) |
||||
if _, err := reader.Read(authorBuf); err != nil { |
||||
return err |
||||
} |
||||
ev.AuthorSerial = decodeUint40(authorBuf) |
||||
|
||||
// Kind
|
||||
kindBuf := make([]byte, 2) |
||||
if _, err := reader.Read(kindBuf); err != nil { |
||||
return err |
||||
} |
||||
ev.Kind = binary.BigEndian.Uint16(kindBuf) |
||||
|
||||
// P-tags
|
||||
ptagCountBuf := make([]byte, 2) |
||||
if _, err := reader.Read(ptagCountBuf); err != nil { |
||||
return err |
||||
} |
||||
ptagCount := binary.BigEndian.Uint16(ptagCountBuf) |
||||
ev.PTagSerials = make([]uint64, ptagCount) |
||||
for i := uint16(0); i < ptagCount; i++ { |
||||
serialBuf := make([]byte, 5) |
||||
if _, err := reader.Read(serialBuf); err != nil { |
||||
return err |
||||
} |
||||
ev.PTagSerials[i] = decodeUint40(serialBuf) |
||||
} |
||||
|
||||
// E-tags
|
||||
etagCountBuf := make([]byte, 2) |
||||
if _, err := reader.Read(etagCountBuf); err != nil { |
||||
return err |
||||
} |
||||
etagCount := binary.BigEndian.Uint16(etagCountBuf) |
||||
ev.ETagSerials = make([]uint64, etagCount) |
||||
for i := uint16(0); i < etagCount; i++ { |
||||
serialBuf := make([]byte, 5) |
||||
if _, err := reader.Read(serialBuf); err != nil { |
||||
return err |
||||
} |
||||
ev.ETagSerials[i] = decodeUint40(serialBuf) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// PubkeyVertex stores the adjacency list for a pubkey.
|
||||
// Contains all events authored by or mentioning this pubkey.
|
||||
type PubkeyVertex struct { |
||||
AuthoredEvents []uint64 // Event serials this pubkey authored
|
||||
MentionedIn []uint64 // Event serials that mention this pubkey (p-tags)
|
||||
} |
||||
|
||||
// Encode serializes the PubkeyVertex to bytes.
|
||||
// Format: authored_count(varint) | [serials(5)...] | mentioned_count(varint) | [serials(5)...]
|
||||
func (pv *PubkeyVertex) Encode() []byte { |
||||
size := 2 + len(pv.AuthoredEvents)*5 + 2 + len(pv.MentionedIn)*5 |
||||
buf := make([]byte, 0, size) |
||||
|
||||
// Authored events
|
||||
authoredCountBuf := make([]byte, 2) |
||||
binary.BigEndian.PutUint16(authoredCountBuf, uint16(len(pv.AuthoredEvents))) |
||||
buf = append(buf, authoredCountBuf...) |
||||
for _, serial := range pv.AuthoredEvents { |
||||
serialBuf := make([]byte, 5) |
||||
encodeUint40(serial, serialBuf) |
||||
buf = append(buf, serialBuf...) |
||||
} |
||||
|
||||
// Mentioned in events
|
||||
mentionedCountBuf := make([]byte, 2) |
||||
binary.BigEndian.PutUint16(mentionedCountBuf, uint16(len(pv.MentionedIn))) |
||||
buf = append(buf, mentionedCountBuf...) |
||||
for _, serial := range pv.MentionedIn { |
||||
serialBuf := make([]byte, 5) |
||||
encodeUint40(serial, serialBuf) |
||||
buf = append(buf, serialBuf...) |
||||
} |
||||
|
||||
return buf |
||||
} |
||||
|
||||
// Decode deserializes bytes into a PubkeyVertex.
|
||||
func (pv *PubkeyVertex) Decode(data []byte) error { |
||||
if len(data) < 4 { // minimum: authored_count(2) + mentioned_count(2)
|
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
|
||||
reader := bytes.NewReader(data) |
||||
|
||||
// Authored events
|
||||
authoredCountBuf := make([]byte, 2) |
||||
if _, err := reader.Read(authoredCountBuf); err != nil { |
||||
return err |
||||
} |
||||
authoredCount := binary.BigEndian.Uint16(authoredCountBuf) |
||||
pv.AuthoredEvents = make([]uint64, authoredCount) |
||||
for i := uint16(0); i < authoredCount; i++ { |
||||
serialBuf := make([]byte, 5) |
||||
if _, err := reader.Read(serialBuf); err != nil { |
||||
return err |
||||
} |
||||
pv.AuthoredEvents[i] = decodeUint40(serialBuf) |
||||
} |
||||
|
||||
// Mentioned in events
|
||||
mentionedCountBuf := make([]byte, 2) |
||||
if _, err := reader.Read(mentionedCountBuf); err != nil { |
||||
return err |
||||
} |
||||
mentionedCount := binary.BigEndian.Uint16(mentionedCountBuf) |
||||
pv.MentionedIn = make([]uint64, mentionedCount) |
||||
for i := uint16(0); i < mentionedCount; i++ { |
||||
serialBuf := make([]byte, 5) |
||||
if _, err := reader.Read(serialBuf); err != nil { |
||||
return err |
||||
} |
||||
pv.MentionedIn[i] = decodeUint40(serialBuf) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// AddAuthored adds an event serial to the authored list if not already present.
|
||||
func (pv *PubkeyVertex) AddAuthored(eventSerial uint64) { |
||||
for _, s := range pv.AuthoredEvents { |
||||
if s == eventSerial { |
||||
return |
||||
} |
||||
} |
||||
pv.AuthoredEvents = append(pv.AuthoredEvents, eventSerial) |
||||
} |
||||
|
||||
// AddMention adds an event serial to the mentioned list if not already present.
|
||||
func (pv *PubkeyVertex) AddMention(eventSerial uint64) { |
||||
for _, s := range pv.MentionedIn { |
||||
if s == eventSerial { |
||||
return |
||||
} |
||||
} |
||||
pv.MentionedIn = append(pv.MentionedIn, eventSerial) |
||||
} |
||||
|
||||
// RemoveAuthored removes an event serial from the authored list.
|
||||
func (pv *PubkeyVertex) RemoveAuthored(eventSerial uint64) { |
||||
for i, s := range pv.AuthoredEvents { |
||||
if s == eventSerial { |
||||
pv.AuthoredEvents = append(pv.AuthoredEvents[:i], pv.AuthoredEvents[i+1:]...) |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
// RemoveMention removes an event serial from the mentioned list.
|
||||
func (pv *PubkeyVertex) RemoveMention(eventSerial uint64) { |
||||
for i, s := range pv.MentionedIn { |
||||
if s == eventSerial { |
||||
pv.MentionedIn = append(pv.MentionedIn[:i], pv.MentionedIn[i+1:]...) |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
// HasAuthored checks if the pubkey authored the given event.
|
||||
func (pv *PubkeyVertex) HasAuthored(eventSerial uint64) bool { |
||||
for _, s := range pv.AuthoredEvents { |
||||
if s == eventSerial { |
||||
return true |
||||
} |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// IsMentionedIn checks if the pubkey is mentioned in the given event.
|
||||
func (pv *PubkeyVertex) IsMentionedIn(eventSerial uint64) bool { |
||||
for _, s := range pv.MentionedIn { |
||||
if s == eventSerial { |
||||
return true |
||||
} |
||||
} |
||||
return false |
||||
} |
||||
@ -1,119 +0,0 @@
@@ -1,119 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
) |
||||
|
||||
// encodeUint40 encodes a uint64 as 5 bytes (big-endian, truncated to 40 bits)
|
||||
func encodeUint40(v uint64, buf []byte) { |
||||
buf[0] = byte(v >> 32) |
||||
buf[1] = byte(v >> 24) |
||||
buf[2] = byte(v >> 16) |
||||
buf[3] = byte(v >> 8) |
||||
buf[4] = byte(v) |
||||
} |
||||
|
||||
// decodeUint40 decodes 5 bytes as uint64
|
||||
func decodeUint40(buf []byte) uint64 { |
||||
return uint64(buf[0])<<32 | |
||||
uint64(buf[1])<<24 | |
||||
uint64(buf[2])<<16 | |
||||
uint64(buf[3])<<8 | |
||||
uint64(buf[4]) |
||||
} |
||||
|
||||
// encodeUint64 encodes a uint64 as 8 bytes (big-endian)
|
||||
func encodeUint64(v uint64, buf []byte) { |
||||
binary.BigEndian.PutUint64(buf, v) |
||||
} |
||||
|
||||
// decodeUint64 decodes 8 bytes as uint64
|
||||
func decodeUint64(buf []byte) uint64 { |
||||
return binary.BigEndian.Uint64(buf) |
||||
} |
||||
|
||||
// encodeUint32 encodes a uint32 as 4 bytes (big-endian)
|
||||
func encodeUint32(v uint32, buf []byte) { |
||||
binary.BigEndian.PutUint32(buf, v) |
||||
} |
||||
|
||||
// decodeUint32 decodes 4 bytes as uint32
|
||||
func decodeUint32(buf []byte) uint32 { |
||||
return binary.BigEndian.Uint32(buf) |
||||
} |
||||
|
||||
// encodeUint16 encodes a uint16 as 2 bytes (big-endian)
|
||||
func encodeUint16(v uint16, buf []byte) { |
||||
binary.BigEndian.PutUint16(buf, v) |
||||
} |
||||
|
||||
// decodeUint16 decodes 2 bytes as uint16
|
||||
func decodeUint16(buf []byte) uint16 { |
||||
return binary.BigEndian.Uint16(buf) |
||||
} |
||||
|
||||
// encodeVarint encodes a uint64 as a variable-length integer
|
||||
// Returns the number of bytes written
|
||||
func encodeVarint(v uint64, buf []byte) int { |
||||
return binary.PutUvarint(buf, v) |
||||
} |
||||
|
||||
// decodeVarint decodes a variable-length integer
|
||||
// Returns the value and the number of bytes read
|
||||
func decodeVarint(buf []byte) (uint64, int) { |
||||
return binary.Uvarint(buf) |
||||
} |
||||
|
||||
// makeSerialKey creates a 5-byte key from a serial number
|
||||
func makeSerialKey(serial uint64) []byte { |
||||
key := make([]byte, 5) |
||||
encodeUint40(serial, key) |
||||
return key |
||||
} |
||||
|
||||
// makePubkeyHashKey creates an 8-byte key from a pubkey hash
|
||||
func makePubkeyHashKey(hash []byte) []byte { |
||||
key := make([]byte, 8) |
||||
copy(key, hash[:8]) |
||||
return key |
||||
} |
||||
|
||||
// makeIdHashKey creates an 8-byte key from an event ID hash
|
||||
func makeIdHashKey(id []byte) []byte { |
||||
key := make([]byte, 8) |
||||
copy(key, id[:8]) |
||||
return key |
||||
} |
||||
|
||||
// hashPubkey returns the first 8 bytes of a 32-byte pubkey as a hash
|
||||
func hashPubkey(pubkey []byte) []byte { |
||||
if len(pubkey) < 8 { |
||||
return pubkey |
||||
} |
||||
return pubkey[:8] |
||||
} |
||||
|
||||
// hashEventId returns the first 8 bytes of a 32-byte event ID as a hash
|
||||
func hashEventId(id []byte) []byte { |
||||
if len(id) < 8 { |
||||
return id |
||||
} |
||||
return id[:8] |
||||
} |
||||
|
||||
// concatenate joins multiple byte slices into one
|
||||
func concatenate(slices ...[]byte) []byte { |
||||
var totalLen int |
||||
for _, s := range slices { |
||||
totalLen += len(s) |
||||
} |
||||
result := make([]byte, totalLen) |
||||
var offset int |
||||
for _, s := range slices { |
||||
copy(result[offset:], s) |
||||
offset += len(s) |
||||
} |
||||
return result |
||||
} |
||||
@ -1,66 +0,0 @@
@@ -1,66 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"crypto/rand" |
||||
"errors" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
) |
||||
|
||||
const identityKey = "relay_identity_secret" |
||||
|
||||
// GetRelayIdentitySecret gets the relay's identity secret key.
|
||||
func (b *B) GetRelayIdentitySecret() (skb []byte, err error) { |
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return errors.New("bbolt: meta bucket not found") |
||||
} |
||||
data := bucket.Get([]byte(identityKey)) |
||||
if data == nil { |
||||
return errors.New("bbolt: relay identity not set") |
||||
} |
||||
skb = make([]byte, len(data)) |
||||
copy(skb, data) |
||||
return nil |
||||
}) |
||||
return |
||||
} |
||||
|
||||
// SetRelayIdentitySecret sets the relay's identity secret key.
|
||||
func (b *B) SetRelayIdentitySecret(skb []byte) error { |
||||
if len(skb) != 32 { |
||||
return errors.New("bbolt: invalid secret key length (must be 32 bytes)") |
||||
} |
||||
return b.db.Update(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return errors.New("bbolt: meta bucket not found") |
||||
} |
||||
return bucket.Put([]byte(identityKey), skb) |
||||
}) |
||||
} |
||||
|
||||
// GetOrCreateRelayIdentitySecret gets or creates the relay's identity secret.
|
||||
func (b *B) GetOrCreateRelayIdentitySecret() (skb []byte, err error) { |
||||
// Try to get existing secret
|
||||
skb, err = b.GetRelayIdentitySecret() |
||||
if err == nil && len(skb) == 32 { |
||||
return skb, nil |
||||
} |
||||
|
||||
// Generate new secret
|
||||
skb = make([]byte, 32) |
||||
if _, err = rand.Read(skb); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// Store it
|
||||
if err = b.SetRelayIdentitySecret(skb); err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return skb, nil |
||||
} |
||||
@ -1,306 +0,0 @@
@@ -1,306 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"bufio" |
||||
"context" |
||||
"io" |
||||
"os" |
||||
"runtime/debug" |
||||
"strings" |
||||
"time" |
||||
|
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
|
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
) |
||||
|
||||
const maxLen = 500000000 |
||||
|
||||
// ImportEventsFromReader imports events from an io.Reader containing JSONL data
|
||||
func (b *B) ImportEventsFromReader(ctx context.Context, rr io.Reader) error { |
||||
startTime := time.Now() |
||||
log.I.F("bbolt import: starting import operation") |
||||
|
||||
// Store to disk so we can return fast
|
||||
tmpPath := os.TempDir() + string(os.PathSeparator) + "orly" |
||||
os.MkdirAll(tmpPath, 0700) |
||||
tmp, err := os.CreateTemp(tmpPath, "") |
||||
if chk.E(err) { |
||||
return err |
||||
} |
||||
tmpName := tmp.Name() |
||||
defer os.Remove(tmpName) // Clean up temp file when done
|
||||
|
||||
log.I.F("bbolt import: buffering upload to %s", tmpName) |
||||
bufferStart := time.Now() |
||||
bytesBuffered, err := io.Copy(tmp, rr) |
||||
if chk.E(err) { |
||||
return err |
||||
} |
||||
bufferElapsed := time.Since(bufferStart) |
||||
log.I.F("bbolt import: buffered %.2f MB in %v (%.2f MB/sec)", |
||||
float64(bytesBuffered)/1024/1024, bufferElapsed.Round(time.Millisecond), |
||||
float64(bytesBuffered)/bufferElapsed.Seconds()/1024/1024) |
||||
|
||||
if _, err = tmp.Seek(0, 0); chk.E(err) { |
||||
return err |
||||
} |
||||
|
||||
count, processErr := b.processJSONLEventsReturningCount(ctx, tmp) |
||||
|
||||
// Close temp file to release resources before index building
|
||||
tmp.Close() |
||||
|
||||
if processErr != nil { |
||||
return processErr |
||||
} |
||||
|
||||
// Build indexes after events are stored (minimal import mode)
|
||||
if count > 0 { |
||||
// Force garbage collection to reclaim memory before index building
|
||||
debug.FreeOSMemory() |
||||
log.I.F("bbolt import: building indexes for %d events...", count) |
||||
if err := b.BuildIndexes(ctx); err != nil { |
||||
log.E.F("bbolt import: failed to build indexes: %v", err) |
||||
return err |
||||
} |
||||
} |
||||
|
||||
totalElapsed := time.Since(startTime) |
||||
log.I.F("bbolt import: total operation time: %v", totalElapsed.Round(time.Millisecond)) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// ImportEventsFromStrings imports events from a slice of JSON strings with policy filtering
|
||||
func (b *B) ImportEventsFromStrings(ctx context.Context, eventJSONs []string, policyManager interface { |
||||
CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) |
||||
}) error { |
||||
// Create a reader from the string slice
|
||||
reader := strings.NewReader(strings.Join(eventJSONs, "\n")) |
||||
return b.processJSONLEventsWithPolicy(ctx, reader, policyManager) |
||||
} |
||||
|
||||
// processJSONLEvents processes JSONL events from a reader
|
||||
func (b *B) processJSONLEvents(ctx context.Context, rr io.Reader) error { |
||||
_, err := b.processJSONLEventsReturningCount(ctx, rr) |
||||
return err |
||||
} |
||||
|
||||
// processJSONLEventsReturningCount processes JSONL events and returns the count saved
|
||||
// This is used by ImportEventsFromReader for migration mode (minimal import without inline indexes)
|
||||
func (b *B) processJSONLEventsReturningCount(ctx context.Context, rr io.Reader) (int, error) { |
||||
// Create a scanner to read the buffer line by line
|
||||
scan := bufio.NewScanner(rr) |
||||
scanBuf := make([]byte, maxLen) |
||||
scan.Buffer(scanBuf, maxLen) |
||||
|
||||
// Performance tracking
|
||||
startTime := time.Now() |
||||
lastLogTime := startTime |
||||
const logInterval = 5 * time.Second |
||||
|
||||
var count, total, skipped, unmarshalErrors, saveErrors int |
||||
for scan.Scan() { |
||||
select { |
||||
case <-ctx.Done(): |
||||
log.I.F("bbolt import: context closed after %d events", count) |
||||
return count, ctx.Err() |
||||
default: |
||||
} |
||||
|
||||
line := scan.Bytes() |
||||
total += len(line) + 1 |
||||
if len(line) < 1 { |
||||
skipped++ |
||||
continue |
||||
} |
||||
|
||||
ev := event.New() |
||||
if _, err := ev.Unmarshal(line); err != nil { |
||||
ev.Free() |
||||
unmarshalErrors++ |
||||
log.W.F("bbolt import: failed to unmarshal event: %v", err) |
||||
continue |
||||
} |
||||
|
||||
// Minimal path for migration: store events only, indexes built later
|
||||
if err := b.SaveEventMinimal(ev); err != nil { |
||||
ev.Free() |
||||
saveErrors++ |
||||
log.W.F("bbolt import: failed to save event: %v", err) |
||||
continue |
||||
} |
||||
|
||||
ev.Free() |
||||
line = nil |
||||
count++ |
||||
|
||||
// Progress logging every logInterval
|
||||
if time.Since(lastLogTime) >= logInterval { |
||||
elapsed := time.Since(startTime) |
||||
eventsPerSec := float64(count) / elapsed.Seconds() |
||||
mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024 |
||||
log.I.F("bbolt import: progress %d events saved, %.2f MB read, %.0f events/sec, %.2f MB/sec", |
||||
count, float64(total)/1024/1024, eventsPerSec, mbPerSec) |
||||
lastLogTime = time.Now() |
||||
debug.FreeOSMemory() |
||||
} |
||||
} |
||||
|
||||
// Flush any remaining batched events
|
||||
if b.batcher != nil { |
||||
b.batcher.Flush() |
||||
} |
||||
|
||||
// Final summary
|
||||
elapsed := time.Since(startTime) |
||||
eventsPerSec := float64(count) / elapsed.Seconds() |
||||
mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024 |
||||
log.I.F("bbolt import: completed - %d events saved, %.2f MB in %v (%.0f events/sec, %.2f MB/sec)", |
||||
count, float64(total)/1024/1024, elapsed.Round(time.Millisecond), eventsPerSec, mbPerSec) |
||||
if unmarshalErrors > 0 || saveErrors > 0 || skipped > 0 { |
||||
log.I.F("bbolt import: stats - %d unmarshal errors, %d save errors, %d skipped empty lines", |
||||
unmarshalErrors, saveErrors, skipped) |
||||
} |
||||
|
||||
if err := scan.Err(); err != nil { |
||||
return count, err |
||||
} |
||||
|
||||
// Clear scanner buffer to help GC
|
||||
scanBuf = nil |
||||
|
||||
return count, nil |
||||
} |
||||
|
||||
// processJSONLEventsWithPolicy processes JSONL events from a reader with optional policy filtering
|
||||
func (b *B) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, policyManager interface { |
||||
CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) |
||||
}) error { |
||||
// Create a scanner to read the buffer line by line
|
||||
scan := bufio.NewScanner(rr) |
||||
scanBuf := make([]byte, maxLen) |
||||
scan.Buffer(scanBuf, maxLen) |
||||
|
||||
// Performance tracking
|
||||
startTime := time.Now() |
||||
lastLogTime := startTime |
||||
const logInterval = 5 * time.Second |
||||
|
||||
var count, total, skipped, policyRejected, unmarshalErrors, saveErrors int |
||||
for scan.Scan() { |
||||
select { |
||||
case <-ctx.Done(): |
||||
log.I.F("bbolt import: context closed after %d events", count) |
||||
return ctx.Err() |
||||
default: |
||||
} |
||||
|
||||
line := scan.Bytes() |
||||
total += len(line) + 1 |
||||
if len(line) < 1 { |
||||
skipped++ |
||||
continue |
||||
} |
||||
|
||||
ev := event.New() |
||||
if _, err := ev.Unmarshal(line); err != nil { |
||||
// return the pooled buffer on error
|
||||
ev.Free() |
||||
unmarshalErrors++ |
||||
log.W.F("bbolt import: failed to unmarshal event: %v", err) |
||||
continue |
||||
} |
||||
|
||||
// Apply policy checking if policy manager is provided
|
||||
if policyManager != nil { |
||||
// For sync imports, we treat events as coming from system/trusted source
|
||||
// Use nil pubkey and empty remote to indicate system-level import
|
||||
allowed, policyErr := policyManager.CheckPolicy("write", ev, nil, "") |
||||
if policyErr != nil { |
||||
log.W.F("bbolt import: policy check failed for event %x: %v", ev.ID, policyErr) |
||||
ev.Free() |
||||
policyRejected++ |
||||
continue |
||||
} |
||||
if !allowed { |
||||
log.D.F("bbolt import: policy rejected event %x during sync import", ev.ID) |
||||
ev.Free() |
||||
policyRejected++ |
||||
continue |
||||
} |
||||
log.D.F("bbolt import: policy allowed event %x during sync import", ev.ID) |
||||
// With policy checking, use regular SaveEvent path
|
||||
if _, err := b.SaveEvent(ctx, ev); err != nil { |
||||
ev.Free() |
||||
saveErrors++ |
||||
log.W.F("bbolt import: failed to save event: %v", err) |
||||
continue |
||||
} |
||||
} else { |
||||
// Minimal path for migration: store events only, build indexes later
|
||||
if err := b.SaveEventMinimal(ev); err != nil { |
||||
ev.Free() |
||||
saveErrors++ |
||||
log.W.F("bbolt import: failed to save event: %v", err) |
||||
continue |
||||
} |
||||
} |
||||
|
||||
// return the pooled buffer after successful save
|
||||
ev.Free() |
||||
line = nil |
||||
count++ |
||||
|
||||
// Progress logging every logInterval
|
||||
if time.Since(lastLogTime) >= logInterval { |
||||
elapsed := time.Since(startTime) |
||||
eventsPerSec := float64(count) / elapsed.Seconds() |
||||
mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024 |
||||
log.I.F("bbolt import: progress %d events saved, %.2f MB read, %.0f events/sec, %.2f MB/sec", |
||||
count, float64(total)/1024/1024, eventsPerSec, mbPerSec) |
||||
lastLogTime = time.Now() |
||||
debug.FreeOSMemory() |
||||
} |
||||
} |
||||
|
||||
// Flush any remaining batched events
|
||||
if b.batcher != nil { |
||||
b.batcher.Flush() |
||||
} |
||||
|
||||
// Final summary
|
||||
elapsed := time.Since(startTime) |
||||
eventsPerSec := float64(count) / elapsed.Seconds() |
||||
mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024 |
||||
log.I.F("bbolt import: completed - %d events saved, %.2f MB in %v (%.0f events/sec, %.2f MB/sec)", |
||||
count, float64(total)/1024/1024, elapsed.Round(time.Millisecond), eventsPerSec, mbPerSec) |
||||
if unmarshalErrors > 0 || saveErrors > 0 || policyRejected > 0 || skipped > 0 { |
||||
log.I.F("bbolt import: stats - %d unmarshal errors, %d save errors, %d policy rejected, %d skipped empty lines", |
||||
unmarshalErrors, saveErrors, policyRejected, skipped) |
||||
} |
||||
|
||||
if err := scan.Err(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Import imports events from a reader (legacy interface).
|
||||
func (b *B) Import(rr io.Reader) { |
||||
ctx := context.Background() |
||||
if err := b.ImportEventsFromReader(ctx, rr); err != nil { |
||||
log.E.F("bbolt import: error: %v", err) |
||||
} |
||||
} |
||||
|
||||
// Export exports events to a writer.
|
||||
func (b *B) Export(c context.Context, w io.Writer, pubkeys ...[]byte) { |
||||
// TODO: Implement export functionality
|
||||
log.W.F("bbolt export: not yet implemented") |
||||
} |
||||
@ -1,232 +0,0 @@
@@ -1,232 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"errors" |
||||
"runtime/debug" |
||||
"sort" |
||||
"time" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
"next.orly.dev/pkg/database" |
||||
"next.orly.dev/pkg/database/bufpool" |
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
) |
||||
|
||||
// SaveEventMinimal stores only the essential event data for fast bulk import.
|
||||
// It skips all indexes - call BuildIndexes after import completes.
|
||||
func (b *B) SaveEventMinimal(ev *event.E) error { |
||||
if ev == nil { |
||||
return errors.New("nil event") |
||||
} |
||||
|
||||
// Reject ephemeral events
|
||||
if ev.Kind >= 20000 && ev.Kind <= 29999 { |
||||
return nil |
||||
} |
||||
|
||||
// Get the next serial number
|
||||
serial := b.getNextEventSerial() |
||||
|
||||
// Serialize event in raw binary format (not compact - preserves full pubkey)
|
||||
// This allows index building to work without pubkey serial resolution
|
||||
legacyBuf := bufpool.GetMedium() |
||||
defer bufpool.PutMedium(legacyBuf) |
||||
ev.MarshalBinary(legacyBuf) |
||||
eventData := bufpool.CopyBytes(legacyBuf) |
||||
|
||||
// Create minimal batch - only event data and ID mappings
|
||||
batch := &EventBatch{ |
||||
Serial: serial, |
||||
EventData: eventData, |
||||
Indexes: []BatchedWrite{ |
||||
// Event ID -> Serial (for lookups)
|
||||
{BucketName: bucketEid, Key: ev.ID[:], Value: makeSerialKey(serial)}, |
||||
// Serial -> Event ID (for reverse lookups)
|
||||
{BucketName: bucketSei, Key: makeSerialKey(serial), Value: ev.ID[:]}, |
||||
}, |
||||
} |
||||
|
||||
return b.batcher.Add(batch) |
||||
} |
||||
|
||||
// BuildIndexes builds all query indexes from stored events.
|
||||
// Call this after importing events with SaveEventMinimal.
|
||||
// Processes events in chunks to avoid OOM on large databases.
|
||||
func (b *B) BuildIndexes(ctx context.Context) error { |
||||
log.I.F("bbolt: starting index build...") |
||||
startTime := time.Now() |
||||
|
||||
// Force GC before starting to reclaim batch buffer memory
|
||||
debug.FreeOSMemory() |
||||
|
||||
// Process in small chunks to avoid OOM on memory-constrained systems
|
||||
// With ~15 indexes per event and ~50 bytes per key, 50k events = ~37.5MB per chunk
|
||||
const chunkSize = 50000 |
||||
|
||||
var totalEvents int |
||||
var lastSerial uint64 = 0 |
||||
var lastLogTime = time.Now() |
||||
|
||||
for { |
||||
select { |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
default: |
||||
} |
||||
|
||||
// Collect indexes for this chunk
|
||||
indexesByBucket := make(map[string][][]byte) |
||||
var chunkEvents int |
||||
var chunkSerial uint64 |
||||
|
||||
// Read a chunk of events
|
||||
err := b.db.View(func(tx *bolt.Tx) error { |
||||
cmpBucket := tx.Bucket(bucketCmp) |
||||
if cmpBucket == nil { |
||||
return errors.New("cmp bucket not found") |
||||
} |
||||
|
||||
cursor := cmpBucket.Cursor() |
||||
|
||||
// Seek to start position
|
||||
var k, v []byte |
||||
if lastSerial == 0 { |
||||
k, v = cursor.First() |
||||
} else { |
||||
// Seek past the last processed serial
|
||||
seekKey := makeSerialKey(lastSerial + 1) |
||||
k, v = cursor.Seek(seekKey) |
||||
} |
||||
|
||||
for ; k != nil && chunkEvents < chunkSize; k, v = cursor.Next() { |
||||
serial := decodeSerialKey(k) |
||||
chunkSerial = serial |
||||
|
||||
// Decode event from raw binary format
|
||||
ev := event.New() |
||||
if err := ev.UnmarshalBinary(bytes.NewBuffer(v)); err != nil { |
||||
log.W.F("bbolt: failed to unmarshal event at serial %d: %v", serial, err) |
||||
continue |
||||
} |
||||
|
||||
// Generate indexes for this event
|
||||
rawIdxs, err := database.GetIndexesForEvent(ev, serial) |
||||
if chk.E(err) { |
||||
ev.Free() |
||||
continue |
||||
} |
||||
|
||||
// Group by bucket (first 3 bytes)
|
||||
for _, idx := range rawIdxs { |
||||
if len(idx) < 3 { |
||||
continue |
||||
} |
||||
bucketName := string(idx[:3]) |
||||
key := idx[3:] |
||||
|
||||
// Skip eid and sei - already stored during import
|
||||
if bucketName == "eid" || bucketName == "sei" { |
||||
continue |
||||
} |
||||
|
||||
// Make a copy of the key
|
||||
keyCopy := make([]byte, len(key)) |
||||
copy(keyCopy, key) |
||||
indexesByBucket[bucketName] = append(indexesByBucket[bucketName], keyCopy) |
||||
} |
||||
|
||||
ev.Free() |
||||
chunkEvents++ |
||||
} |
||||
return nil |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// No more events to process
|
||||
if chunkEvents == 0 { |
||||
break |
||||
} |
||||
|
||||
totalEvents += chunkEvents |
||||
lastSerial = chunkSerial |
||||
|
||||
// Progress logging
|
||||
if time.Since(lastLogTime) >= 5*time.Second { |
||||
log.I.F("bbolt: index build progress: %d events processed", totalEvents) |
||||
lastLogTime = time.Now() |
||||
} |
||||
|
||||
// Count total keys in this chunk
|
||||
var totalKeys int |
||||
for _, keys := range indexesByBucket { |
||||
totalKeys += len(keys) |
||||
} |
||||
log.I.F("bbolt: writing %d index keys for chunk (%d events)", totalKeys, chunkEvents) |
||||
|
||||
// Write this chunk's indexes
|
||||
for bucketName, keys := range indexesByBucket { |
||||
if len(keys) == 0 { |
||||
continue |
||||
} |
||||
|
||||
bucketBytes := []byte(bucketName) |
||||
|
||||
// Sort keys for this bucket before writing
|
||||
sort.Slice(keys, func(i, j int) bool { |
||||
return bytes.Compare(keys[i], keys[j]) < 0 |
||||
}) |
||||
|
||||
// Write in batches
|
||||
const batchSize = 50000 |
||||
for i := 0; i < len(keys); i += batchSize { |
||||
end := i + batchSize |
||||
if end > len(keys) { |
||||
end = len(keys) |
||||
} |
||||
batch := keys[i:end] |
||||
|
||||
err := b.db.Update(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketBytes) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
for _, key := range batch { |
||||
if err := bucket.Put(key, nil); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
if err != nil { |
||||
log.E.F("bbolt: failed to write batch for bucket %s: %v", bucketName, err) |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Clear for next chunk and release memory
|
||||
indexesByBucket = nil |
||||
debug.FreeOSMemory() |
||||
} |
||||
|
||||
elapsed := time.Since(startTime) |
||||
log.I.F("bbolt: index build complete in %v (%d events)", elapsed.Round(time.Second), totalEvents) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// decodeSerialKey decodes a 5-byte serial key to uint64
|
||||
func decodeSerialKey(b []byte) uint64 { |
||||
if len(b) < 5 { |
||||
return 0 |
||||
} |
||||
return uint64(b[0])<<32 | uint64(b[1])<<24 | uint64(b[2])<<16 | uint64(b[3])<<8 | uint64(b[4]) |
||||
} |
||||
@ -1,55 +0,0 @@
@@ -1,55 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"next.orly.dev/pkg/database" |
||||
) |
||||
|
||||
func init() { |
||||
database.RegisterBboltFactory(newBboltFromConfig) |
||||
} |
||||
|
||||
// newBboltFromConfig creates a BBolt database from DatabaseConfig
|
||||
func newBboltFromConfig( |
||||
ctx context.Context, |
||||
cancel context.CancelFunc, |
||||
cfg *database.DatabaseConfig, |
||||
) (database.Database, error) { |
||||
// Convert DatabaseConfig to BboltConfig
|
||||
bboltCfg := &BboltConfig{ |
||||
DataDir: cfg.DataDir, |
||||
LogLevel: cfg.LogLevel, |
||||
|
||||
// Use bbolt-specific settings from DatabaseConfig if present
|
||||
// These will be added to DatabaseConfig later
|
||||
BatchMaxEvents: cfg.BboltBatchMaxEvents, |
||||
BatchMaxBytes: cfg.BboltBatchMaxBytes, |
||||
BatchFlushTimeout: cfg.BboltFlushTimeout, |
||||
BloomSizeMB: cfg.BboltBloomSizeMB, |
||||
NoSync: cfg.BboltNoSync, |
||||
InitialMmapSize: cfg.BboltMmapSize, |
||||
} |
||||
|
||||
// Apply defaults if not set
|
||||
if bboltCfg.BatchMaxEvents <= 0 { |
||||
bboltCfg.BatchMaxEvents = 5000 |
||||
} |
||||
if bboltCfg.BatchMaxBytes <= 0 { |
||||
bboltCfg.BatchMaxBytes = 128 * 1024 * 1024 // 128MB
|
||||
} |
||||
if bboltCfg.BatchFlushTimeout <= 0 { |
||||
bboltCfg.BatchFlushTimeout = 30 * time.Second |
||||
} |
||||
if bboltCfg.BloomSizeMB <= 0 { |
||||
bboltCfg.BloomSizeMB = 16 |
||||
} |
||||
if bboltCfg.InitialMmapSize <= 0 { |
||||
bboltCfg.InitialMmapSize = 8 * 1024 * 1024 * 1024 // 8GB
|
||||
} |
||||
|
||||
return NewWithConfig(ctx, cancel, bboltCfg) |
||||
} |
||||
@ -1,81 +0,0 @@
@@ -1,81 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"fmt" |
||||
"runtime" |
||||
"strings" |
||||
|
||||
"go.uber.org/atomic" |
||||
"lol.mleku.dev" |
||||
"lol.mleku.dev/log" |
||||
) |
||||
|
||||
// Logger wraps the lol logger for BBolt
|
||||
type Logger struct { |
||||
Level atomic.Int32 |
||||
Label string |
||||
} |
||||
|
||||
// NewLogger creates a new Logger instance
|
||||
func NewLogger(level int, dataDir string) *Logger { |
||||
l := &Logger{Label: "bbolt"} |
||||
l.Level.Store(int32(level)) |
||||
return l |
||||
} |
||||
|
||||
// SetLogLevel updates the log level
|
||||
func (l *Logger) SetLogLevel(level int) { |
||||
l.Level.Store(int32(level)) |
||||
} |
||||
|
||||
// Tracef logs a trace message
|
||||
func (l *Logger) Tracef(format string, args ...interface{}) { |
||||
if l.Level.Load() >= int32(lol.Trace) { |
||||
s := l.Label + ": " + format |
||||
txt := fmt.Sprintf(s, args...) |
||||
_, file, line, _ := runtime.Caller(2) |
||||
log.T.F("%s\n%s:%d", strings.TrimSpace(txt), file, line) |
||||
} |
||||
} |
||||
|
||||
// Debugf logs a debug message
|
||||
func (l *Logger) Debugf(format string, args ...interface{}) { |
||||
if l.Level.Load() >= int32(lol.Debug) { |
||||
s := l.Label + ": " + format |
||||
txt := fmt.Sprintf(s, args...) |
||||
_, file, line, _ := runtime.Caller(2) |
||||
log.D.F("%s\n%s:%d", strings.TrimSpace(txt), file, line) |
||||
} |
||||
} |
||||
|
||||
// Infof logs an info message
|
||||
func (l *Logger) Infof(format string, args ...interface{}) { |
||||
if l.Level.Load() >= int32(lol.Info) { |
||||
s := l.Label + ": " + format |
||||
txt := fmt.Sprintf(s, args...) |
||||
_, file, line, _ := runtime.Caller(2) |
||||
log.I.F("%s\n%s:%d", strings.TrimSpace(txt), file, line) |
||||
} |
||||
} |
||||
|
||||
// Warningf logs a warning message
|
||||
func (l *Logger) Warningf(format string, args ...interface{}) { |
||||
if l.Level.Load() >= int32(lol.Warn) { |
||||
s := l.Label + ": " + format |
||||
txt := fmt.Sprintf(s, args...) |
||||
_, file, line, _ := runtime.Caller(2) |
||||
log.W.F("%s\n%s:%d", strings.TrimSpace(txt), file, line) |
||||
} |
||||
} |
||||
|
||||
// Errorf logs an error message
|
||||
func (l *Logger) Errorf(format string, args ...interface{}) { |
||||
if l.Level.Load() >= int32(lol.Error) { |
||||
s := l.Label + ": " + format |
||||
txt := fmt.Sprintf(s, args...) |
||||
_, file, line, _ := runtime.Caller(2) |
||||
log.E.F("%s\n%s:%d", strings.TrimSpace(txt), file, line) |
||||
} |
||||
} |
||||
@ -1,62 +0,0 @@
@@ -1,62 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
bolt "go.etcd.io/bbolt" |
||||
) |
||||
|
||||
const markerPrefix = "marker:" |
||||
|
||||
// SetMarker sets a metadata marker.
|
||||
func (b *B) SetMarker(key string, value []byte) error { |
||||
return b.db.Update(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
return bucket.Put([]byte(markerPrefix+key), value) |
||||
}) |
||||
} |
||||
|
||||
// GetMarker gets a metadata marker.
|
||||
func (b *B) GetMarker(key string) (value []byte, err error) { |
||||
err = b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
data := bucket.Get([]byte(markerPrefix + key)) |
||||
if data != nil { |
||||
value = make([]byte, len(data)) |
||||
copy(value, data) |
||||
} |
||||
return nil |
||||
}) |
||||
return |
||||
} |
||||
|
||||
// HasMarker checks if a marker exists.
|
||||
func (b *B) HasMarker(key string) bool { |
||||
var exists bool |
||||
b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
exists = bucket.Get([]byte(markerPrefix+key)) != nil |
||||
return nil |
||||
}) |
||||
return exists |
||||
} |
||||
|
||||
// DeleteMarker deletes a marker.
|
||||
func (b *B) DeleteMarker(key string) error { |
||||
return b.db.Update(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
return bucket.Delete([]byte(markerPrefix + key)) |
||||
}) |
||||
} |
||||
@ -1,287 +0,0 @@
@@ -1,287 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"bytes" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"next.orly.dev/pkg/database/indexes/types" |
||||
) |
||||
|
||||
// EdgeExists checks if an edge exists between two serials.
|
||||
// Uses bloom filter for fast negative lookups.
|
||||
func (b *B) EdgeExists(srcSerial, dstSerial uint64, edgeType byte) (bool, error) { |
||||
// Fast path: check bloom filter first
|
||||
if !b.edgeBloom.MayExist(srcSerial, dstSerial, edgeType) { |
||||
return false, nil // Definitely doesn't exist
|
||||
} |
||||
|
||||
// Bloom says maybe - need to verify in adjacency list
|
||||
return b.verifyEdgeInAdjacencyList(srcSerial, dstSerial, edgeType) |
||||
} |
||||
|
||||
// verifyEdgeInAdjacencyList checks the adjacency list for edge existence.
|
||||
func (b *B) verifyEdgeInAdjacencyList(srcSerial, dstSerial uint64, edgeType byte) (bool, error) { |
||||
var exists bool |
||||
|
||||
err := b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketEv) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
key := makeSerialKey(srcSerial) |
||||
data := bucket.Get(key) |
||||
if data == nil { |
||||
return nil |
||||
} |
||||
|
||||
vertex := &EventVertex{} |
||||
if err := vertex.Decode(data); err != nil { |
||||
return err |
||||
} |
||||
|
||||
switch edgeType { |
||||
case EdgeTypeAuthor: |
||||
exists = vertex.AuthorSerial == dstSerial |
||||
case EdgeTypePTag: |
||||
for _, s := range vertex.PTagSerials { |
||||
if s == dstSerial { |
||||
exists = true |
||||
break |
||||
} |
||||
} |
||||
case EdgeTypeETag: |
||||
for _, s := range vertex.ETagSerials { |
||||
if s == dstSerial { |
||||
exists = true |
||||
break |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
return exists, err |
||||
} |
||||
|
||||
// GetEventVertex retrieves the adjacency list for an event.
|
||||
func (b *B) GetEventVertex(eventSerial uint64) (*EventVertex, error) { |
||||
var vertex *EventVertex |
||||
|
||||
err := b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketEv) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
key := makeSerialKey(eventSerial) |
||||
data := bucket.Get(key) |
||||
if data == nil { |
||||
return nil |
||||
} |
||||
|
||||
vertex = &EventVertex{} |
||||
return vertex.Decode(data) |
||||
}) |
||||
|
||||
return vertex, err |
||||
} |
||||
|
||||
// GetPubkeyVertex retrieves the adjacency list for a pubkey.
|
||||
func (b *B) GetPubkeyVertex(pubkeySerial uint64) (*PubkeyVertex, error) { |
||||
var vertex *PubkeyVertex |
||||
|
||||
err := b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketPv) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
key := makeSerialKey(pubkeySerial) |
||||
data := bucket.Get(key) |
||||
if data == nil { |
||||
return nil |
||||
} |
||||
|
||||
vertex = &PubkeyVertex{} |
||||
return vertex.Decode(data) |
||||
}) |
||||
|
||||
return vertex, err |
||||
} |
||||
|
||||
// GetEventsAuthoredBy returns event serials authored by a pubkey.
|
||||
func (b *B) GetEventsAuthoredBy(pubkeySerial uint64) ([]uint64, error) { |
||||
vertex, err := b.GetPubkeyVertex(pubkeySerial) |
||||
if err != nil || vertex == nil { |
||||
return nil, err |
||||
} |
||||
return vertex.AuthoredEvents, nil |
||||
} |
||||
|
||||
// GetEventsMentioning returns event serials that mention a pubkey.
|
||||
func (b *B) GetEventsMentioning(pubkeySerial uint64) ([]uint64, error) { |
||||
vertex, err := b.GetPubkeyVertex(pubkeySerial) |
||||
if err != nil || vertex == nil { |
||||
return nil, err |
||||
} |
||||
return vertex.MentionedIn, nil |
||||
} |
||||
|
||||
// GetPTagsFromEvent returns pubkey serials tagged in an event.
|
||||
func (b *B) GetPTagsFromEvent(eventSerial uint64) ([]uint64, error) { |
||||
vertex, err := b.GetEventVertex(eventSerial) |
||||
if err != nil || vertex == nil { |
||||
return nil, err |
||||
} |
||||
return vertex.PTagSerials, nil |
||||
} |
||||
|
||||
// GetETagsFromEvent returns event serials referenced by an event.
|
||||
func (b *B) GetETagsFromEvent(eventSerial uint64) ([]uint64, error) { |
||||
vertex, err := b.GetEventVertex(eventSerial) |
||||
if err != nil || vertex == nil { |
||||
return nil, err |
||||
} |
||||
return vertex.ETagSerials, nil |
||||
} |
||||
|
||||
// GetFollowsFromPubkeySerial returns the pubkey serials that a user follows.
|
||||
// This extracts p-tags from the user's kind-3 contact list event.
|
||||
func (b *B) GetFollowsFromPubkeySerial(pubkeySerial *types.Uint40) ([]*types.Uint40, error) { |
||||
if pubkeySerial == nil { |
||||
return nil, nil |
||||
} |
||||
|
||||
// Find the kind-3 event for this pubkey
|
||||
contactEventSerial, err := b.FindEventByAuthorAndKind(pubkeySerial.Get(), 3) |
||||
if err != nil { |
||||
return nil, nil // No kind-3 event found is not an error
|
||||
} |
||||
if contactEventSerial == 0 { |
||||
return nil, nil |
||||
} |
||||
|
||||
// Get the p-tags from the event vertex
|
||||
pTagSerials, err := b.GetPTagsFromEvent(contactEventSerial) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// Convert to types.Uint40
|
||||
result := make([]*types.Uint40, 0, len(pTagSerials)) |
||||
for _, s := range pTagSerials { |
||||
ser := new(types.Uint40) |
||||
ser.Set(s) |
||||
result = append(result, ser) |
||||
} |
||||
return result, nil |
||||
} |
||||
|
||||
// FindEventByAuthorAndKind finds an event serial by author and kind.
|
||||
// For replaceable events like kind-3, returns the most recent one.
|
||||
func (b *B) FindEventByAuthorAndKind(authorSerial uint64, kindNum uint16) (uint64, error) { |
||||
var resultSerial uint64 |
||||
|
||||
err := b.db.View(func(tx *bolt.Tx) error { |
||||
// First, get events authored by this pubkey
|
||||
pvBucket := tx.Bucket(bucketPv) |
||||
if pvBucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
pvKey := makeSerialKey(authorSerial) |
||||
pvData := pvBucket.Get(pvKey) |
||||
if pvData == nil { |
||||
return nil |
||||
} |
||||
|
||||
vertex := &PubkeyVertex{} |
||||
if err := vertex.Decode(pvData); err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Search through authored events for matching kind
|
||||
evBucket := tx.Bucket(bucketEv) |
||||
if evBucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
var latestTs int64 |
||||
for _, eventSerial := range vertex.AuthoredEvents { |
||||
evKey := makeSerialKey(eventSerial) |
||||
evData := evBucket.Get(evKey) |
||||
if evData == nil { |
||||
continue |
||||
} |
||||
|
||||
evVertex := &EventVertex{} |
||||
if err := evVertex.Decode(evData); err != nil { |
||||
continue |
||||
} |
||||
|
||||
if evVertex.Kind == kindNum { |
||||
// For replaceable events, we need to check timestamp
|
||||
// Get event to compare timestamps
|
||||
fpcBucket := tx.Bucket(bucketFpc) |
||||
if fpcBucket != nil { |
||||
// Scan for matching serial prefix in fpc bucket
|
||||
c := fpcBucket.Cursor() |
||||
prefix := makeSerialKey(eventSerial) |
||||
for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() { |
||||
// Key format: serial(5) | id(32) | pubkey_hash(8) | created_at(8)
|
||||
if len(k) >= 53 { |
||||
ts := int64(decodeUint64(k[45:53])) |
||||
if ts > latestTs { |
||||
latestTs = ts |
||||
resultSerial = eventSerial |
||||
} |
||||
} |
||||
break |
||||
} |
||||
} else { |
||||
// If no fpc bucket, just take the first match
|
||||
resultSerial = eventSerial |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
return resultSerial, err |
||||
} |
||||
|
||||
// GetReferencingEvents returns event serials that reference a target event via e-tag.
|
||||
func (b *B) GetReferencingEvents(targetSerial uint64) ([]uint64, error) { |
||||
var result []uint64 |
||||
|
||||
err := b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketEv) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
// Scan all event vertices looking for e-tag references
|
||||
// Note: This is O(n) - for production, consider a reverse index
|
||||
c := bucket.Cursor() |
||||
for k, v := c.First(); k != nil; k, v = c.Next() { |
||||
vertex := &EventVertex{} |
||||
if err := vertex.Decode(v); err != nil { |
||||
continue |
||||
} |
||||
|
||||
for _, eTagSerial := range vertex.ETagSerials { |
||||
if eTagSerial == targetSerial { |
||||
eventSerial := decodeUint40(k) |
||||
result = append(result, eventSerial) |
||||
break |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
|
||||
return result, err |
||||
} |
||||
@ -1,96 +0,0 @@
@@ -1,96 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"errors" |
||||
|
||||
"lol.mleku.dev/chk" |
||||
"next.orly.dev/pkg/database" |
||||
"next.orly.dev/pkg/database/bufpool" |
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
) |
||||
|
||||
// SaveEventForImport saves an event optimized for bulk import.
|
||||
// It skips duplicate checking, deletion checking, and graph vertex creation
|
||||
// to maximize import throughput. Use only for trusted data migration.
|
||||
func (b *B) SaveEventForImport(ev *event.E) error { |
||||
if ev == nil { |
||||
return errors.New("nil event") |
||||
} |
||||
|
||||
// Reject ephemeral events (kinds 20000-29999)
|
||||
if ev.Kind >= 20000 && ev.Kind <= 29999 { |
||||
return nil // silently skip
|
||||
} |
||||
|
||||
// Get the next serial number
|
||||
serial := b.getNextEventSerial() |
||||
|
||||
// Generate all indexes using the shared function
|
||||
rawIdxs, err := database.GetIndexesForEvent(ev, serial) |
||||
if chk.E(err) { |
||||
return err |
||||
} |
||||
|
||||
// Convert raw indexes to BatchedWrites, stripping the 3-byte prefix
|
||||
batch := &EventBatch{ |
||||
Serial: serial, |
||||
Indexes: make([]BatchedWrite, 0, len(rawIdxs)+1), |
||||
} |
||||
|
||||
for _, idx := range rawIdxs { |
||||
if len(idx) < 3 { |
||||
continue |
||||
} |
||||
bucketName := idx[:3] |
||||
key := idx[3:] |
||||
batch.Indexes = append(batch.Indexes, BatchedWrite{ |
||||
BucketName: bucketName, |
||||
Key: key, |
||||
Value: nil, |
||||
}) |
||||
} |
||||
|
||||
// Serialize event in compact format (without graph references for import)
|
||||
resolver := &nullSerialResolver{} |
||||
compactData, compactErr := database.MarshalCompactEvent(ev, resolver) |
||||
if compactErr != nil { |
||||
// Fall back to legacy format
|
||||
legacyBuf := bufpool.GetMedium() |
||||
defer bufpool.PutMedium(legacyBuf) |
||||
ev.MarshalBinary(legacyBuf) |
||||
compactData = bufpool.CopyBytes(legacyBuf) |
||||
} |
||||
batch.EventData = compactData |
||||
|
||||
// Store serial -> event ID mapping
|
||||
batch.Indexes = append(batch.Indexes, BatchedWrite{ |
||||
BucketName: bucketSei, |
||||
Key: makeSerialKey(serial), |
||||
Value: ev.ID[:], |
||||
}) |
||||
|
||||
// Add to batcher (no graph vertex, no pubkey lookups)
|
||||
return b.batcher.Add(batch) |
||||
} |
||||
|
||||
// nullSerialResolver returns 0 for all lookups, used for fast import
|
||||
// where we don't need pubkey/event serial references in compact format
|
||||
type nullSerialResolver struct{} |
||||
|
||||
func (r *nullSerialResolver) GetOrCreatePubkeySerial(pubkey []byte) (uint64, error) { |
||||
return 0, nil |
||||
} |
||||
|
||||
func (r *nullSerialResolver) GetPubkeyBySerial(serial uint64) ([]byte, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (r *nullSerialResolver) GetEventSerialById(eventID []byte) (uint64, bool, error) { |
||||
return 0, false, nil |
||||
} |
||||
|
||||
func (r *nullSerialResolver) GetEventIdBySerial(serial uint64) ([]byte, error) { |
||||
return nil, nil |
||||
} |
||||
@ -1,393 +0,0 @@
@@ -1,393 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"strings" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"lol.mleku.dev/chk" |
||||
"next.orly.dev/pkg/database" |
||||
"next.orly.dev/pkg/database/bufpool" |
||||
"next.orly.dev/pkg/database/indexes/types" |
||||
"next.orly.dev/pkg/mode" |
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
"git.mleku.dev/mleku/nostr/encoders/filter" |
||||
"git.mleku.dev/mleku/nostr/encoders/hex" |
||||
"git.mleku.dev/mleku/nostr/encoders/kind" |
||||
"git.mleku.dev/mleku/nostr/encoders/tag" |
||||
) |
||||
|
||||
// SaveEvent saves an event to the database using the write batcher.
|
||||
func (b *B) SaveEvent(c context.Context, ev *event.E) (replaced bool, err error) { |
||||
if ev == nil { |
||||
err = errors.New("nil event") |
||||
return |
||||
} |
||||
|
||||
// Reject ephemeral events (kinds 20000-29999)
|
||||
if ev.Kind >= 20000 && ev.Kind <= 29999 { |
||||
err = errors.New("blocked: ephemeral events should not be stored") |
||||
return |
||||
} |
||||
|
||||
// Validate kind 3 (follow list) events have at least one p tag
|
||||
if ev.Kind == 3 { |
||||
hasPTag := false |
||||
if ev.Tags != nil { |
||||
for _, tag := range *ev.Tags { |
||||
if tag != nil && tag.Len() >= 2 { |
||||
key := tag.Key() |
||||
if len(key) == 1 && key[0] == 'p' { |
||||
hasPTag = true |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
if !hasPTag { |
||||
err = errors.New("blocked: kind 3 follow list events must have at least one p tag") |
||||
return |
||||
} |
||||
} |
||||
|
||||
// Check if the event already exists
|
||||
var ser *types.Uint40 |
||||
if ser, err = b.GetSerialById(ev.ID); err == nil && ser != nil { |
||||
err = errors.New("blocked: event already exists: " + hex.Enc(ev.ID[:])) |
||||
return |
||||
} |
||||
|
||||
// If the error is "id not found", we can proceed
|
||||
if err != nil && strings.Contains(err.Error(), "id not found") { |
||||
err = nil |
||||
} else if err != nil { |
||||
return |
||||
} |
||||
|
||||
// Check if the event has been deleted
|
||||
if !mode.IsOpen() { |
||||
if err = b.CheckForDeleted(ev, nil); err != nil { |
||||
err = fmt.Errorf("blocked: %s", err.Error()) |
||||
return |
||||
} |
||||
} |
||||
|
||||
// Check for replacement
|
||||
if kind.IsReplaceable(ev.Kind) || kind.IsParameterizedReplaceable(ev.Kind) { |
||||
var werr error |
||||
if replaced, _, werr = b.WouldReplaceEvent(ev); werr != nil { |
||||
if errors.Is(werr, database.ErrOlderThanExisting) { |
||||
if kind.IsReplaceable(ev.Kind) { |
||||
err = errors.New("blocked: event is older than existing replaceable event") |
||||
} else { |
||||
err = errors.New("blocked: event is older than existing addressable event") |
||||
} |
||||
return |
||||
} |
||||
if errors.Is(werr, database.ErrMissingDTag) { |
||||
err = database.ErrMissingDTag |
||||
return |
||||
} |
||||
return |
||||
} |
||||
} |
||||
|
||||
// Get the next serial number
|
||||
serial := b.getNextEventSerial() |
||||
|
||||
// Generate all indexes using the shared function
|
||||
var rawIdxs [][]byte |
||||
if rawIdxs, err = database.GetIndexesForEvent(ev, serial); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Convert raw indexes to BatchedWrites, stripping the 3-byte prefix
|
||||
// since we use separate buckets
|
||||
batch := &EventBatch{ |
||||
Serial: serial, |
||||
Indexes: make([]BatchedWrite, 0, len(rawIdxs)), |
||||
} |
||||
|
||||
for _, idx := range rawIdxs { |
||||
if len(idx) < 3 { |
||||
continue |
||||
} |
||||
|
||||
// Get bucket name from prefix
|
||||
bucketName := idx[:3] |
||||
key := idx[3:] // Key without prefix
|
||||
|
||||
batch.Indexes = append(batch.Indexes, BatchedWrite{ |
||||
BucketName: bucketName, |
||||
Key: key, |
||||
Value: nil, // Index entries have empty values
|
||||
}) |
||||
} |
||||
|
||||
// Serialize event in compact format
|
||||
resolver := &bboltSerialResolver{b: b} |
||||
compactData, compactErr := database.MarshalCompactEvent(ev, resolver) |
||||
if compactErr != nil { |
||||
// Fall back to legacy format
|
||||
legacyBuf := bufpool.GetMedium() |
||||
defer bufpool.PutMedium(legacyBuf) |
||||
ev.MarshalBinary(legacyBuf) |
||||
compactData = bufpool.CopyBytes(legacyBuf) |
||||
} |
||||
batch.EventData = compactData |
||||
|
||||
// Build event vertex for adjacency list
|
||||
var authorSerial uint64 |
||||
err = b.db.Update(func(tx *bolt.Tx) error { |
||||
var e error |
||||
authorSerial, e = b.getOrCreatePubkeySerial(tx, ev.Pubkey) |
||||
return e |
||||
}) |
||||
if chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
eventVertex := &EventVertex{ |
||||
AuthorSerial: authorSerial, |
||||
Kind: uint16(ev.Kind), |
||||
PTagSerials: make([]uint64, 0), |
||||
ETagSerials: make([]uint64, 0), |
||||
} |
||||
|
||||
// Collect edge keys for bloom filter
|
||||
edgeKeys := make([]EdgeKey, 0) |
||||
|
||||
// Add author edge to bloom filter
|
||||
edgeKeys = append(edgeKeys, EdgeKey{ |
||||
SrcSerial: serial, |
||||
DstSerial: authorSerial, |
||||
EdgeType: EdgeTypeAuthor, |
||||
}) |
||||
|
||||
// Set up pubkey vertex update for author
|
||||
batch.PubkeyUpdate = &PubkeyVertexUpdate{ |
||||
PubkeySerial: authorSerial, |
||||
AddAuthored: serial, |
||||
} |
||||
|
||||
// Process p-tags
|
||||
batch.MentionUpdates = make([]*PubkeyVertexUpdate, 0) |
||||
pTags := ev.Tags.GetAll([]byte("p")) |
||||
for _, pTag := range pTags { |
||||
if pTag.Len() >= 2 { |
||||
var ptagPubkey []byte |
||||
if ptagPubkey, err = hex.Dec(string(pTag.ValueHex())); err == nil && len(ptagPubkey) == 32 { |
||||
var ptagSerial uint64 |
||||
err = b.db.Update(func(tx *bolt.Tx) error { |
||||
var e error |
||||
ptagSerial, e = b.getOrCreatePubkeySerial(tx, ptagPubkey) |
||||
return e |
||||
}) |
||||
if chk.E(err) { |
||||
continue |
||||
} |
||||
|
||||
eventVertex.PTagSerials = append(eventVertex.PTagSerials, ptagSerial) |
||||
|
||||
// Add p-tag edge to bloom filter
|
||||
edgeKeys = append(edgeKeys, EdgeKey{ |
||||
SrcSerial: serial, |
||||
DstSerial: ptagSerial, |
||||
EdgeType: EdgeTypePTag, |
||||
}) |
||||
|
||||
// Add mention update for this pubkey
|
||||
batch.MentionUpdates = append(batch.MentionUpdates, &PubkeyVertexUpdate{ |
||||
PubkeySerial: ptagSerial, |
||||
AddMention: serial, |
||||
}) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Process e-tags
|
||||
eTags := ev.Tags.GetAll([]byte("e")) |
||||
for _, eTag := range eTags { |
||||
if eTag.Len() >= 2 { |
||||
var targetEventID []byte |
||||
if targetEventID, err = hex.Dec(string(eTag.ValueHex())); err != nil || len(targetEventID) != 32 { |
||||
continue |
||||
} |
||||
|
||||
// Look up the target event's serial
|
||||
var targetSerial *types.Uint40 |
||||
if targetSerial, err = b.GetSerialById(targetEventID); err != nil { |
||||
err = nil |
||||
continue |
||||
} |
||||
|
||||
targetSer := targetSerial.Get() |
||||
eventVertex.ETagSerials = append(eventVertex.ETagSerials, targetSer) |
||||
|
||||
// Add e-tag edge to bloom filter
|
||||
edgeKeys = append(edgeKeys, EdgeKey{ |
||||
SrcSerial: serial, |
||||
DstSerial: targetSer, |
||||
EdgeType: EdgeTypeETag, |
||||
}) |
||||
} |
||||
} |
||||
|
||||
batch.EventVertex = eventVertex |
||||
batch.EdgeKeys = edgeKeys |
||||
|
||||
// Store serial -> event ID mapping
|
||||
batch.Indexes = append(batch.Indexes, BatchedWrite{ |
||||
BucketName: bucketSei, |
||||
Key: makeSerialKey(serial), |
||||
Value: ev.ID[:], |
||||
}) |
||||
|
||||
// Add to batcher
|
||||
if err = b.batcher.Add(batch); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Process deletion events
|
||||
if ev.Kind == kind.Deletion.K { |
||||
if err = b.ProcessDelete(ev, nil); chk.E(err) { |
||||
b.Logger.Warningf("failed to process deletion for event %x: %v", ev.ID, err) |
||||
err = nil |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
// GetSerialsFromFilter returns serials matching a filter.
|
||||
func (b *B) GetSerialsFromFilter(f *filter.F) (sers types.Uint40s, err error) { |
||||
var idxs []database.Range |
||||
if idxs, err = database.GetIndexesFromFilter(f); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
sers = make(types.Uint40s, 0, len(idxs)*100) |
||||
for _, idx := range idxs { |
||||
var s types.Uint40s |
||||
if s, err = b.GetSerialsByRange(idx); chk.E(err) { |
||||
continue |
||||
} |
||||
sers = append(sers, s...) |
||||
} |
||||
return |
||||
} |
||||
|
||||
// WouldReplaceEvent checks if the event would replace existing events.
|
||||
func (b *B) WouldReplaceEvent(ev *event.E) (bool, types.Uint40s, error) { |
||||
if !(kind.IsReplaceable(ev.Kind) || kind.IsParameterizedReplaceable(ev.Kind)) { |
||||
return false, nil, nil |
||||
} |
||||
|
||||
var f *filter.F |
||||
if kind.IsReplaceable(ev.Kind) { |
||||
f = &filter.F{ |
||||
Authors: tag.NewFromBytesSlice(ev.Pubkey), |
||||
Kinds: kind.NewS(kind.New(ev.Kind)), |
||||
} |
||||
} else { |
||||
dTag := ev.Tags.GetFirst([]byte("d")) |
||||
if dTag == nil { |
||||
return false, nil, database.ErrMissingDTag |
||||
} |
||||
f = &filter.F{ |
||||
Authors: tag.NewFromBytesSlice(ev.Pubkey), |
||||
Kinds: kind.NewS(kind.New(ev.Kind)), |
||||
Tags: tag.NewS( |
||||
tag.NewFromAny("d", dTag.Value()), |
||||
), |
||||
} |
||||
} |
||||
|
||||
sers, err := b.GetSerialsFromFilter(f) |
||||
if chk.E(err) { |
||||
return false, nil, err |
||||
} |
||||
if len(sers) == 0 { |
||||
return false, nil, nil |
||||
} |
||||
|
||||
shouldReplace := true |
||||
for _, s := range sers { |
||||
oldEv, ferr := b.FetchEventBySerial(s) |
||||
if chk.E(ferr) { |
||||
continue |
||||
} |
||||
if ev.CreatedAt < oldEv.CreatedAt { |
||||
shouldReplace = false |
||||
break |
||||
} |
||||
} |
||||
if shouldReplace { |
||||
return true, nil, nil |
||||
} |
||||
return false, nil, database.ErrOlderThanExisting |
||||
} |
||||
|
||||
// bboltSerialResolver implements database.SerialResolver for compact event encoding
|
||||
type bboltSerialResolver struct { |
||||
b *B |
||||
} |
||||
|
||||
func (r *bboltSerialResolver) GetOrCreatePubkeySerial(pubkey []byte) (serial uint64, err error) { |
||||
err = r.b.db.Update(func(tx *bolt.Tx) error { |
||||
var e error |
||||
serial, e = r.b.getOrCreatePubkeySerial(tx, pubkey) |
||||
return e |
||||
}) |
||||
return |
||||
} |
||||
|
||||
func (r *bboltSerialResolver) GetPubkeyBySerial(serial uint64) (pubkey []byte, err error) { |
||||
r.b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketSpk) |
||||
if bucket == nil { |
||||
err = errors.New("bbolt: spk bucket not found") |
||||
return nil |
||||
} |
||||
val := bucket.Get(makeSerialKey(serial)) |
||||
if val != nil { |
||||
pubkey = make([]byte, 32) |
||||
copy(pubkey, val) |
||||
} else { |
||||
err = errors.New("bbolt: pubkey serial not found") |
||||
} |
||||
return nil |
||||
}) |
||||
return |
||||
} |
||||
|
||||
func (r *bboltSerialResolver) GetEventSerialById(eventID []byte) (serial uint64, found bool, err error) { |
||||
ser, e := r.b.GetSerialById(eventID) |
||||
if e != nil || ser == nil { |
||||
return 0, false, nil |
||||
} |
||||
return ser.Get(), true, nil |
||||
} |
||||
|
||||
func (r *bboltSerialResolver) GetEventIdBySerial(serial uint64) (eventID []byte, err error) { |
||||
r.b.db.View(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketSei) |
||||
if bucket == nil { |
||||
err = errors.New("bbolt: sei bucket not found") |
||||
return nil |
||||
} |
||||
val := bucket.Get(makeSerialKey(serial)) |
||||
if val != nil { |
||||
eventID = make([]byte, 32) |
||||
copy(eventID, val) |
||||
} else { |
||||
err = errors.New("bbolt: event serial not found") |
||||
} |
||||
return nil |
||||
}) |
||||
return |
||||
} |
||||
@ -1,169 +0,0 @@
@@ -1,169 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
|
||||
bolt "go.etcd.io/bbolt" |
||||
"lol.mleku.dev/chk" |
||||
) |
||||
|
||||
const ( |
||||
serialCounterKey = "serial_counter" |
||||
pubkeySerialCounterKey = "pubkey_serial_counter" |
||||
) |
||||
|
||||
// initSerialCounters initializes or loads the serial counters from _meta bucket
|
||||
func (b *B) initSerialCounters() error { |
||||
return b.db.Update(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
// Load event serial counter
|
||||
val := bucket.Get([]byte(serialCounterKey)) |
||||
if val == nil { |
||||
b.nextSerial = 1 |
||||
buf := make([]byte, 8) |
||||
binary.BigEndian.PutUint64(buf, 1) |
||||
if err := bucket.Put([]byte(serialCounterKey), buf); err != nil { |
||||
return err |
||||
} |
||||
} else { |
||||
b.nextSerial = binary.BigEndian.Uint64(val) |
||||
} |
||||
|
||||
// Load pubkey serial counter
|
||||
val = bucket.Get([]byte(pubkeySerialCounterKey)) |
||||
if val == nil { |
||||
b.nextPubkeySeq = 1 |
||||
buf := make([]byte, 8) |
||||
binary.BigEndian.PutUint64(buf, 1) |
||||
if err := bucket.Put([]byte(pubkeySerialCounterKey), buf); err != nil { |
||||
return err |
||||
} |
||||
} else { |
||||
b.nextPubkeySeq = binary.BigEndian.Uint64(val) |
||||
} |
||||
|
||||
return nil |
||||
}) |
||||
} |
||||
|
||||
// persistSerialCounters saves the current serial counters to disk
|
||||
func (b *B) persistSerialCounters() error { |
||||
b.serialMu.Lock() |
||||
eventSerial := b.nextSerial |
||||
pubkeySerial := b.nextPubkeySeq |
||||
b.serialMu.Unlock() |
||||
|
||||
return b.db.Update(func(tx *bolt.Tx) error { |
||||
bucket := tx.Bucket(bucketMeta) |
||||
if bucket == nil { |
||||
return nil |
||||
} |
||||
|
||||
buf := make([]byte, 8) |
||||
binary.BigEndian.PutUint64(buf, eventSerial) |
||||
if err := bucket.Put([]byte(serialCounterKey), buf); chk.E(err) { |
||||
return err |
||||
} |
||||
|
||||
binary.BigEndian.PutUint64(buf, pubkeySerial) |
||||
if err := bucket.Put([]byte(pubkeySerialCounterKey), buf); chk.E(err) { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
}) |
||||
} |
||||
|
||||
// getNextEventSerial returns the next event serial number (thread-safe)
|
||||
func (b *B) getNextEventSerial() uint64 { |
||||
b.serialMu.Lock() |
||||
defer b.serialMu.Unlock() |
||||
|
||||
serial := b.nextSerial |
||||
b.nextSerial++ |
||||
|
||||
// Persist every 1000 to reduce disk writes
|
||||
if b.nextSerial%1000 == 0 { |
||||
go func() { |
||||
if err := b.persistSerialCounters(); chk.E(err) { |
||||
b.Logger.Warningf("bbolt: failed to persist serial counters: %v", err) |
||||
} |
||||
}() |
||||
} |
||||
|
||||
return serial |
||||
} |
||||
|
||||
// getNextPubkeySerial returns the next pubkey serial number (thread-safe)
|
||||
func (b *B) getNextPubkeySerial() uint64 { |
||||
b.serialMu.Lock() |
||||
defer b.serialMu.Unlock() |
||||
|
||||
serial := b.nextPubkeySeq |
||||
b.nextPubkeySeq++ |
||||
|
||||
// Persist every 1000 to reduce disk writes
|
||||
if b.nextPubkeySeq%1000 == 0 { |
||||
go func() { |
||||
if err := b.persistSerialCounters(); chk.E(err) { |
||||
b.Logger.Warningf("bbolt: failed to persist serial counters: %v", err) |
||||
} |
||||
}() |
||||
} |
||||
|
||||
return serial |
||||
} |
||||
|
||||
// getOrCreatePubkeySerial gets or creates a serial for a pubkey
|
||||
func (b *B) getOrCreatePubkeySerial(tx *bolt.Tx, pubkey []byte) (uint64, error) { |
||||
pksBucket := tx.Bucket(bucketPks) |
||||
spkBucket := tx.Bucket(bucketSpk) |
||||
|
||||
if pksBucket == nil || spkBucket == nil { |
||||
return 0, nil |
||||
} |
||||
|
||||
// Check if pubkey already has a serial
|
||||
pubkeyHash := hashPubkey(pubkey) |
||||
val := pksBucket.Get(pubkeyHash) |
||||
if val != nil { |
||||
return decodeUint40(val), nil |
||||
} |
||||
|
||||
// Create new serial
|
||||
serial := b.getNextPubkeySerial() |
||||
serialKey := makeSerialKey(serial) |
||||
|
||||
// Store pubkey_hash -> serial
|
||||
serialBuf := make([]byte, 5) |
||||
encodeUint40(serial, serialBuf) |
||||
if err := pksBucket.Put(pubkeyHash, serialBuf); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
// Store serial -> full pubkey
|
||||
fullPubkey := make([]byte, 32) |
||||
copy(fullPubkey, pubkey) |
||||
if err := spkBucket.Put(serialKey, fullPubkey); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
return serial, nil |
||||
} |
||||
|
||||
// getPubkeyBySerial retrieves the full 32-byte pubkey from a serial
|
||||
func (b *B) getPubkeyBySerial(tx *bolt.Tx, serial uint64) ([]byte, error) { |
||||
spkBucket := tx.Bucket(bucketSpk) |
||||
if spkBucket == nil { |
||||
return nil, nil |
||||
} |
||||
|
||||
serialKey := makeSerialKey(serial) |
||||
return spkBucket.Get(serialKey), nil |
||||
} |
||||
@ -1,233 +0,0 @@
@@ -1,233 +0,0 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package bbolt |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"time" |
||||
|
||||
"next.orly.dev/pkg/database" |
||||
"next.orly.dev/pkg/database/indexes/types" |
||||
"next.orly.dev/pkg/interfaces/store" |
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
"git.mleku.dev/mleku/nostr/encoders/filter" |
||||
) |
||||
|
||||
// This file contains stub implementations for interface methods that will be
|
||||
// implemented fully in later phases. It allows the code to compile while
|
||||
// we implement the core functionality.
|
||||
|
||||
var errNotImplemented = errors.New("bbolt: not implemented yet") |
||||
|
||||
// QueryEvents queries events matching a filter.
|
||||
func (b *B) QueryEvents(c context.Context, f *filter.F) (evs event.S, err error) { |
||||
// Get serials matching filter
|
||||
var serials types.Uint40s |
||||
if serials, err = b.GetSerialsFromFilter(f); err != nil { |
||||
return |
||||
} |
||||
|
||||
// Fetch events by serials
|
||||
evs = make(event.S, 0, len(serials)) |
||||
for _, ser := range serials { |
||||
ev, e := b.FetchEventBySerial(ser) |
||||
if e == nil && ev != nil { |
||||
evs = append(evs, ev) |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// QueryAllVersions queries all versions of events matching a filter.
|
||||
func (b *B) QueryAllVersions(c context.Context, f *filter.F) (evs event.S, err error) { |
||||
return b.QueryEvents(c, f) |
||||
} |
||||
|
||||
// QueryEventsWithOptions queries events with additional options.
|
||||
func (b *B) QueryEventsWithOptions(c context.Context, f *filter.F, includeDeleteEvents bool, showAllVersions bool) (evs event.S, err error) { |
||||
return b.QueryEvents(c, f) |
||||
} |
||||
|
||||
// QueryDeleteEventsByTargetId queries delete events targeting a specific event.
|
||||
func (b *B) QueryDeleteEventsByTargetId(c context.Context, targetEventId []byte) (evs event.S, err error) { |
||||
return nil, errNotImplemented |
||||
} |
||||
|
||||
// QueryForSerials queries and returns only the serials.
|
||||
func (b *B) QueryForSerials(c context.Context, f *filter.F) (serials types.Uint40s, err error) { |
||||
return b.GetSerialsFromFilter(f) |
||||
} |
||||
|
||||
// QueryForIds queries and returns event ID/pubkey/timestamp tuples.
|
||||
func (b *B) QueryForIds(c context.Context, f *filter.F) (idPkTs []*store.IdPkTs, err error) { |
||||
var serials types.Uint40s |
||||
if serials, err = b.GetSerialsFromFilter(f); err != nil { |
||||
return |
||||
} |
||||
return b.GetFullIdPubkeyBySerials(serials) |
||||
} |
||||
|
||||
// DeleteEvent deletes an event by ID.
|
||||
func (b *B) DeleteEvent(c context.Context, eid []byte) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// DeleteEventBySerial deletes an event by serial.
|
||||
func (b *B) DeleteEventBySerial(c context.Context, ser *types.Uint40, ev *event.E) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// DeleteExpired deletes expired events.
|
||||
func (b *B) DeleteExpired() { |
||||
// TODO: Implement
|
||||
} |
||||
|
||||
// ProcessDelete processes a deletion event.
|
||||
// For migration from other backends, deletions have already been processed,
|
||||
// so this is a no-op. Full implementation needed for production use.
|
||||
func (b *B) ProcessDelete(ev *event.E, admins [][]byte) error { |
||||
// TODO: Implement full deletion processing for production use
|
||||
// For now, just return nil to allow migrations to proceed
|
||||
return nil |
||||
} |
||||
|
||||
// CheckForDeleted checks if an event has been deleted.
|
||||
func (b *B) CheckForDeleted(ev *event.E, admins [][]byte) error { |
||||
return nil // Not deleted by default
|
||||
} |
||||
|
||||
// GetSubscription gets a user's subscription.
|
||||
func (b *B) GetSubscription(pubkey []byte) (*database.Subscription, error) { |
||||
return nil, errNotImplemented |
||||
} |
||||
|
||||
// IsSubscriptionActive checks if a subscription is active.
|
||||
func (b *B) IsSubscriptionActive(pubkey []byte) (bool, error) { |
||||
return false, errNotImplemented |
||||
} |
||||
|
||||
// ExtendSubscription extends a subscription.
|
||||
func (b *B) ExtendSubscription(pubkey []byte, days int) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// RecordPayment records a payment.
|
||||
func (b *B) RecordPayment(pubkey []byte, amount int64, invoice, preimage string) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// GetPaymentHistory gets payment history.
|
||||
func (b *B) GetPaymentHistory(pubkey []byte) ([]database.Payment, error) { |
||||
return nil, errNotImplemented |
||||
} |
||||
|
||||
// ExtendBlossomSubscription extends a Blossom subscription.
|
||||
func (b *B) ExtendBlossomSubscription(pubkey []byte, tier string, storageMB int64, daysExtended int) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// GetBlossomStorageQuota gets Blossom storage quota.
|
||||
func (b *B) GetBlossomStorageQuota(pubkey []byte) (quotaMB int64, err error) { |
||||
return 0, errNotImplemented |
||||
} |
||||
|
||||
// IsFirstTimeUser checks if this is a first-time user.
|
||||
func (b *B) IsFirstTimeUser(pubkey []byte) (bool, error) { |
||||
return true, nil |
||||
} |
||||
|
||||
// AddNIP43Member adds a NIP-43 member.
|
||||
func (b *B) AddNIP43Member(pubkey []byte, inviteCode string) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// RemoveNIP43Member removes a NIP-43 member.
|
||||
func (b *B) RemoveNIP43Member(pubkey []byte) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// IsNIP43Member checks if pubkey is a NIP-43 member.
|
||||
func (b *B) IsNIP43Member(pubkey []byte) (isMember bool, err error) { |
||||
return false, errNotImplemented |
||||
} |
||||
|
||||
// GetNIP43Membership gets NIP-43 membership details.
|
||||
func (b *B) GetNIP43Membership(pubkey []byte) (*database.NIP43Membership, error) { |
||||
return nil, errNotImplemented |
||||
} |
||||
|
||||
// GetAllNIP43Members gets all NIP-43 members.
|
||||
func (b *B) GetAllNIP43Members() ([][]byte, error) { |
||||
return nil, errNotImplemented |
||||
} |
||||
|
||||
// StoreInviteCode stores an invite code.
|
||||
func (b *B) StoreInviteCode(code string, expiresAt time.Time) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// ValidateInviteCode validates an invite code.
|
||||
func (b *B) ValidateInviteCode(code string) (valid bool, err error) { |
||||
return false, errNotImplemented |
||||
} |
||||
|
||||
// DeleteInviteCode deletes an invite code.
|
||||
func (b *B) DeleteInviteCode(code string) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// PublishNIP43MembershipEvent publishes a NIP-43 membership event.
|
||||
func (b *B) PublishNIP43MembershipEvent(kind int, pubkey []byte) error { |
||||
return errNotImplemented |
||||
} |
||||
|
||||
// RunMigrations runs database migrations.
|
||||
func (b *B) RunMigrations() { |
||||
// TODO: Implement if needed
|
||||
} |
||||
|
||||
// GetCachedJSON gets cached JSON for a filter (stub - no caching in bbolt).
|
||||
func (b *B) GetCachedJSON(f *filter.F) ([][]byte, bool) { |
||||
return nil, false |
||||
} |
||||
|
||||
// CacheMarshaledJSON caches JSON for a filter (stub - no caching in bbolt).
|
||||
func (b *B) CacheMarshaledJSON(f *filter.F, marshaledJSON [][]byte) { |
||||
// No-op: BBolt doesn't use query cache to save RAM
|
||||
} |
||||
|
||||
// GetCachedEvents gets cached events for a filter (stub - no caching in bbolt).
|
||||
func (b *B) GetCachedEvents(f *filter.F) (event.S, bool) { |
||||
return nil, false |
||||
} |
||||
|
||||
// CacheEvents caches events for a filter (stub - no caching in bbolt).
|
||||
func (b *B) CacheEvents(f *filter.F, events event.S) { |
||||
// No-op: BBolt doesn't use query cache to save RAM
|
||||
} |
||||
|
||||
// InvalidateQueryCache invalidates the query cache (stub - no caching in bbolt).
|
||||
func (b *B) InvalidateQueryCache() { |
||||
// No-op
|
||||
} |
||||
|
||||
// RecordEventAccess records an event access.
|
||||
func (b *B) RecordEventAccess(serial uint64, connectionID string) error { |
||||
return nil // TODO: Implement if needed
|
||||
} |
||||
|
||||
// GetEventAccessInfo gets event access information.
|
||||
func (b *B) GetEventAccessInfo(serial uint64) (lastAccess int64, accessCount uint32, err error) { |
||||
return 0, 0, errNotImplemented |
||||
} |
||||
|
||||
// GetLeastAccessedEvents gets least accessed events.
|
||||
func (b *B) GetLeastAccessedEvents(limit int, minAgeSec int64) (serials []uint64, err error) { |
||||
return nil, errNotImplemented |
||||
} |
||||
|
||||
// EventIdsBySerial gets event IDs by serial range.
|
||||
func (b *B) EventIdsBySerial(start uint64, count int) (evs []uint64, err error) { |
||||
return nil, errNotImplemented |
||||
} |
||||
@ -0,0 +1,569 @@
@@ -0,0 +1,569 @@
|
||||
//go:build !(js && wasm)
|
||||
|
||||
package database |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"os" |
||||
"path/filepath" |
||||
"sort" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/dgraph-io/badger/v4" |
||||
"github.com/minio/sha256-simd" |
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/errorf" |
||||
"lol.mleku.dev/log" |
||||
|
||||
"git.mleku.dev/mleku/nostr/encoders/hex" |
||||
) |
||||
|
||||
const ( |
||||
// Database key prefixes for blob storage (metadata and indexes only, blob data stored as files)
|
||||
prefixBlobMeta = "blob:meta:" |
||||
prefixBlobIndex = "blob:index:" |
||||
prefixBlobReport = "blob:report:" |
||||
) |
||||
|
||||
// getBlobDir returns the directory for storing blob files
|
||||
func (d *D) getBlobDir() string { |
||||
return filepath.Join(d.dataDir, "blossom") |
||||
} |
||||
|
||||
// getBlobPath returns the filesystem path for a blob given its hash and extension
|
||||
func (d *D) getBlobPath(sha256Hex string, ext string) string { |
||||
filename := sha256Hex + ext |
||||
return filepath.Join(d.getBlobDir(), filename) |
||||
} |
||||
|
||||
// ensureBlobDir ensures the blob directory exists
|
||||
func (d *D) ensureBlobDir() error { |
||||
return os.MkdirAll(d.getBlobDir(), 0755) |
||||
} |
||||
|
||||
// SaveBlob stores a blob with its metadata
|
||||
func (d *D) SaveBlob( |
||||
sha256Hash []byte, data []byte, pubkey []byte, mimeType string, extension string, |
||||
) (err error) { |
||||
sha256Hex := hex.Enc(sha256Hash) |
||||
|
||||
// Verify SHA256 matches
|
||||
calculatedHash := sha256.Sum256(data) |
||||
if !bytesEqual(calculatedHash[:], sha256Hash) { |
||||
err = errorf.E( |
||||
"SHA256 mismatch: calculated %x, provided %x", |
||||
calculatedHash[:], sha256Hash, |
||||
) |
||||
return |
||||
} |
||||
|
||||
// If extension not provided, infer from MIME type
|
||||
if extension == "" { |
||||
extension = getExtensionFromMimeType(mimeType) |
||||
} |
||||
|
||||
// Create metadata with extension
|
||||
metadata := &BlobMetadata{ |
||||
Pubkey: pubkey, |
||||
MimeType: mimeType, |
||||
Uploaded: time.Now().Unix(), |
||||
Size: int64(len(data)), |
||||
Extension: extension, |
||||
} |
||||
if mimeType == "" { |
||||
metadata.MimeType = "application/octet-stream" |
||||
} |
||||
|
||||
var metaData []byte |
||||
if metaData, err = json.Marshal(metadata); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Ensure blob directory exists
|
||||
if err = d.ensureBlobDir(); err != nil { |
||||
return errorf.E("failed to create blob directory: %w", err) |
||||
} |
||||
|
||||
// Get blob file path
|
||||
blobPath := d.getBlobPath(sha256Hex, extension) |
||||
|
||||
// Check if blob file already exists (deduplication)
|
||||
if _, err = os.Stat(blobPath); err == nil { |
||||
// File exists, just update metadata and index
|
||||
log.D.F("blob file already exists: %s", blobPath) |
||||
} else if !os.IsNotExist(err) { |
||||
return errorf.E("error checking blob file: %w", err) |
||||
} else { |
||||
// Write blob data to file
|
||||
if err = os.WriteFile(blobPath, data, 0644); chk.E(err) { |
||||
return errorf.E("failed to write blob file: %w", err) |
||||
} |
||||
log.D.F("wrote blob file: %s (%d bytes)", blobPath, len(data)) |
||||
} |
||||
|
||||
// Store metadata and index in database
|
||||
if err = d.Update(func(txn *badger.Txn) error { |
||||
// Store metadata
|
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
if err := txn.Set([]byte(metaKey), metaData); err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Index by pubkey
|
||||
indexKey := prefixBlobIndex + hex.Enc(pubkey) + ":" + sha256Hex |
||||
if err := txn.Set([]byte(indexKey), []byte{1}); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
log.D.F("saved blob %s (%d bytes) for pubkey %s", sha256Hex, len(data), hex.Enc(pubkey)) |
||||
return |
||||
} |
||||
|
||||
// GetBlob retrieves blob data by SHA256 hash
|
||||
func (d *D) GetBlob(sha256Hash []byte) (data []byte, metadata *BlobMetadata, err error) { |
||||
sha256Hex := hex.Enc(sha256Hash) |
||||
|
||||
// Get metadata first to get extension
|
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
item, err := txn.Get([]byte(metaKey)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return item.Value(func(val []byte) error { |
||||
metadata = &BlobMetadata{} |
||||
if err = json.Unmarshal(val, metadata); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
}) |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Read blob data from file
|
||||
blobPath := d.getBlobPath(sha256Hex, metadata.Extension) |
||||
data, err = os.ReadFile(blobPath) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
err = badger.ErrKeyNotFound |
||||
} |
||||
return |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
// HasBlob checks if a blob exists
|
||||
func (d *D) HasBlob(sha256Hash []byte) (exists bool, err error) { |
||||
sha256Hex := hex.Enc(sha256Hash) |
||||
|
||||
// Get metadata to find extension
|
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
var metadata *BlobMetadata |
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
item, err := txn.Get([]byte(metaKey)) |
||||
if err == badger.ErrKeyNotFound { |
||||
return badger.ErrKeyNotFound |
||||
} |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return item.Value(func(val []byte) error { |
||||
metadata = &BlobMetadata{} |
||||
if err = json.Unmarshal(val, metadata); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
}) |
||||
}); err == badger.ErrKeyNotFound { |
||||
exists = false |
||||
return false, nil |
||||
} |
||||
if err != nil { |
||||
return |
||||
} |
||||
|
||||
// Check if file exists
|
||||
blobPath := d.getBlobPath(sha256Hex, metadata.Extension) |
||||
if _, err = os.Stat(blobPath); err == nil { |
||||
exists = true |
||||
return |
||||
} |
||||
if os.IsNotExist(err) { |
||||
exists = false |
||||
err = nil |
||||
return |
||||
} |
||||
return |
||||
} |
||||
|
||||
// DeleteBlob deletes a blob and its metadata
|
||||
func (d *D) DeleteBlob(sha256Hash []byte, pubkey []byte) (err error) { |
||||
sha256Hex := hex.Enc(sha256Hash) |
||||
|
||||
// Get metadata to find extension
|
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
var metadata *BlobMetadata |
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
item, err := txn.Get([]byte(metaKey)) |
||||
if err == badger.ErrKeyNotFound { |
||||
return badger.ErrKeyNotFound |
||||
} |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return item.Value(func(val []byte) error { |
||||
metadata = &BlobMetadata{} |
||||
if err = json.Unmarshal(val, metadata); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
}) |
||||
}); err == badger.ErrKeyNotFound { |
||||
return errorf.E("blob %s not found", sha256Hex) |
||||
} |
||||
if err != nil { |
||||
return |
||||
} |
||||
|
||||
blobPath := d.getBlobPath(sha256Hex, metadata.Extension) |
||||
indexKey := prefixBlobIndex + hex.Enc(pubkey) + ":" + sha256Hex |
||||
|
||||
if err = d.Update(func(txn *badger.Txn) error { |
||||
// Delete metadata
|
||||
if err := txn.Delete([]byte(metaKey)); err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Delete index entry
|
||||
if err := txn.Delete([]byte(indexKey)); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Delete blob file
|
||||
if err = os.Remove(blobPath); err != nil && !os.IsNotExist(err) { |
||||
log.E.F("failed to delete blob file %s: %v", blobPath, err) |
||||
// Don't fail if file doesn't exist
|
||||
} |
||||
|
||||
log.D.F("deleted blob %s for pubkey %s", sha256Hex, hex.Enc(pubkey)) |
||||
return |
||||
} |
||||
|
||||
// ListBlobs lists all blobs for a given pubkey
|
||||
func (d *D) ListBlobs( |
||||
pubkey []byte, since, until int64, |
||||
) (descriptors []*BlobDescriptor, err error) { |
||||
pubkeyHex := hex.Enc(pubkey) |
||||
prefix := prefixBlobIndex + pubkeyHex + ":" |
||||
|
||||
descriptors = make([]*BlobDescriptor, 0) |
||||
|
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
opts := badger.DefaultIteratorOptions |
||||
opts.Prefix = []byte(prefix) |
||||
it := txn.NewIterator(opts) |
||||
defer it.Close() |
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() { |
||||
item := it.Item() |
||||
key := item.Key() |
||||
|
||||
// Extract SHA256 from key: prefixBlobIndex + pubkeyHex + ":" + sha256Hex
|
||||
sha256Hex := string(key[len(prefix):]) |
||||
|
||||
// Get blob metadata
|
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
metaItem, err := txn.Get([]byte(metaKey)) |
||||
if err != nil { |
||||
continue |
||||
} |
||||
|
||||
var metadata *BlobMetadata |
||||
if err = metaItem.Value(func(val []byte) error { |
||||
metadata = &BlobMetadata{} |
||||
if err = json.Unmarshal(val, metadata); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
}); err != nil { |
||||
continue |
||||
} |
||||
|
||||
// Filter by time range
|
||||
if since > 0 && metadata.Uploaded < since { |
||||
continue |
||||
} |
||||
if until > 0 && metadata.Uploaded > until { |
||||
continue |
||||
} |
||||
|
||||
// Verify blob file exists
|
||||
blobPath := d.getBlobPath(sha256Hex, metadata.Extension) |
||||
if _, errGet := os.Stat(blobPath); errGet != nil { |
||||
continue |
||||
} |
||||
|
||||
// Create descriptor (URL will be set by handler)
|
||||
mimeType := metadata.MimeType |
||||
if mimeType == "" { |
||||
mimeType = "application/octet-stream" |
||||
} |
||||
descriptor := &BlobDescriptor{ |
||||
URL: "", // URL will be set by handler
|
||||
SHA256: sha256Hex, |
||||
Size: metadata.Size, |
||||
Type: mimeType, |
||||
Uploaded: metadata.Uploaded, |
||||
} |
||||
|
||||
descriptors = append(descriptors, descriptor) |
||||
} |
||||
|
||||
return nil |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
// GetBlobMetadata retrieves only metadata for a blob
|
||||
func (d *D) GetBlobMetadata(sha256Hash []byte) (metadata *BlobMetadata, err error) { |
||||
sha256Hex := hex.Enc(sha256Hash) |
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
|
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
item, err := txn.Get([]byte(metaKey)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return item.Value(func(val []byte) error { |
||||
metadata = &BlobMetadata{} |
||||
if err = json.Unmarshal(val, metadata); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
}) |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
// GetTotalBlobStorageUsed calculates total storage used by a pubkey in MB
|
||||
func (d *D) GetTotalBlobStorageUsed(pubkey []byte) (totalMB int64, err error) { |
||||
pubkeyHex := hex.Enc(pubkey) |
||||
prefix := prefixBlobIndex + pubkeyHex + ":" |
||||
|
||||
totalBytes := int64(0) |
||||
|
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
opts := badger.DefaultIteratorOptions |
||||
opts.Prefix = []byte(prefix) |
||||
it := txn.NewIterator(opts) |
||||
defer it.Close() |
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() { |
||||
item := it.Item() |
||||
key := item.Key() |
||||
|
||||
// Extract SHA256 from key: prefixBlobIndex + pubkeyHex + ":" + sha256Hex
|
||||
sha256Hex := string(key[len(prefix):]) |
||||
|
||||
// Get blob metadata
|
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
metaItem, err := txn.Get([]byte(metaKey)) |
||||
if err != nil { |
||||
continue |
||||
} |
||||
|
||||
var metadata *BlobMetadata |
||||
if err = metaItem.Value(func(val []byte) error { |
||||
metadata = &BlobMetadata{} |
||||
if err = json.Unmarshal(val, metadata); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
}); err != nil { |
||||
continue |
||||
} |
||||
|
||||
// Verify blob file exists
|
||||
blobPath := d.getBlobPath(sha256Hex, metadata.Extension) |
||||
if _, errGet := os.Stat(blobPath); errGet != nil { |
||||
continue |
||||
} |
||||
|
||||
totalBytes += metadata.Size |
||||
} |
||||
|
||||
return nil |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Convert bytes to MB (rounding up)
|
||||
totalMB = (totalBytes + 1024*1024 - 1) / (1024 * 1024) |
||||
return |
||||
} |
||||
|
||||
// SaveBlobReport stores a report for a blob (BUD-09)
|
||||
func (d *D) SaveBlobReport(sha256Hash []byte, reportData []byte) (err error) { |
||||
sha256Hex := hex.Enc(sha256Hash) |
||||
reportKey := prefixBlobReport + sha256Hex |
||||
|
||||
// Get existing reports
|
||||
var existingReports [][]byte |
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
item, err := txn.Get([]byte(reportKey)) |
||||
if err == badger.ErrKeyNotFound { |
||||
return nil |
||||
} |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return item.Value(func(val []byte) error { |
||||
if err = json.Unmarshal(val, &existingReports); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
}) |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Append new report
|
||||
existingReports = append(existingReports, reportData) |
||||
|
||||
// Store updated reports
|
||||
var reportsData []byte |
||||
if reportsData, err = json.Marshal(existingReports); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
if err = d.Update(func(txn *badger.Txn) error { |
||||
return txn.Set([]byte(reportKey), reportsData) |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
log.D.F("saved report for blob %s", sha256Hex) |
||||
return |
||||
} |
||||
|
||||
// ListAllBlobUserStats returns storage statistics for all users who have uploaded blobs
|
||||
func (d *D) ListAllBlobUserStats() (stats []*UserBlobStats, err error) { |
||||
statsMap := make(map[string]*UserBlobStats) |
||||
|
||||
if err = d.View(func(txn *badger.Txn) error { |
||||
opts := badger.DefaultIteratorOptions |
||||
opts.Prefix = []byte(prefixBlobIndex) |
||||
opts.PrefetchValues = false |
||||
it := txn.NewIterator(opts) |
||||
defer it.Close() |
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() { |
||||
key := string(it.Item().Key()) |
||||
// Key format: blob:index:<pubkey-hex>:<sha256-hex>
|
||||
remainder := key[len(prefixBlobIndex):] |
||||
parts := strings.SplitN(remainder, ":", 2) |
||||
if len(parts) != 2 { |
||||
continue |
||||
} |
||||
pubkeyHex := parts[0] |
||||
sha256Hex := parts[1] |
||||
|
||||
// Get or create stats entry
|
||||
stat, ok := statsMap[pubkeyHex] |
||||
if !ok { |
||||
stat = &UserBlobStats{PubkeyHex: pubkeyHex} |
||||
statsMap[pubkeyHex] = stat |
||||
} |
||||
stat.BlobCount++ |
||||
|
||||
// Get blob size from metadata
|
||||
metaKey := prefixBlobMeta + sha256Hex |
||||
metaItem, errGet := txn.Get([]byte(metaKey)) |
||||
if errGet != nil { |
||||
continue |
||||
} |
||||
metaItem.Value(func(val []byte) error { |
||||
metadata := &BlobMetadata{} |
||||
if errDeser := json.Unmarshal(val, metadata); errDeser == nil { |
||||
stat.TotalSizeBytes += metadata.Size |
||||
} |
||||
return nil |
||||
}) |
||||
} |
||||
return nil |
||||
}); chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
// Convert map to slice
|
||||
stats = make([]*UserBlobStats, 0, len(statsMap)) |
||||
for _, stat := range statsMap { |
||||
stats = append(stats, stat) |
||||
} |
||||
|
||||
// Sort by total size descending
|
||||
sort.Slice(stats, func(i, j int) bool { |
||||
return stats[i].TotalSizeBytes > stats[j].TotalSizeBytes |
||||
}) |
||||
|
||||
return |
||||
} |
||||
|
||||
// getExtensionFromMimeType returns a file extension for a MIME type
|
||||
func getExtensionFromMimeType(mimeType string) string { |
||||
// Common MIME type to extension mapping
|
||||
mimeToExt := map[string]string{ |
||||
"image/png": ".png", |
||||
"image/jpeg": ".jpg", |
||||
"image/gif": ".gif", |
||||
"image/webp": ".webp", |
||||
"image/svg+xml": ".svg", |
||||
"image/bmp": ".bmp", |
||||
"image/tiff": ".tiff", |
||||
"video/mp4": ".mp4", |
||||
"video/webm": ".webm", |
||||
"video/ogg": ".ogv", |
||||
"video/quicktime": ".mov", |
||||
"audio/mpeg": ".mp3", |
||||
"audio/ogg": ".ogg", |
||||
"audio/wav": ".wav", |
||||
"audio/webm": ".weba", |
||||
"audio/flac": ".flac", |
||||
"application/pdf": ".pdf", |
||||
"application/zip": ".zip", |
||||
"text/plain": ".txt", |
||||
"text/html": ".html", |
||||
"text/css": ".css", |
||||
"text/javascript": ".js", |
||||
"application/json": ".json", |
||||
} |
||||
|
||||
if ext, ok := mimeToExt[mimeType]; ok { |
||||
return ext |
||||
} |
||||
return "" // No extension for unknown types
|
||||
} |
||||
@ -0,0 +1,875 @@
@@ -0,0 +1,875 @@
|
||||
// Package grpc provides a gRPC client that implements the database.Database interface.
|
||||
// This allows the relay to use a remote database server via gRPC.
|
||||
package grpc |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
"time" |
||||
|
||||
"google.golang.org/grpc" |
||||
"google.golang.org/grpc/credentials/insecure" |
||||
"lol.mleku.dev/chk" |
||||
"lol.mleku.dev/log" |
||||
|
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
"git.mleku.dev/mleku/nostr/encoders/filter" |
||||
"git.mleku.dev/mleku/nostr/encoders/tag" |
||||
"next.orly.dev/pkg/database" |
||||
indextypes "next.orly.dev/pkg/database/indexes/types" |
||||
"next.orly.dev/pkg/interfaces/store" |
||||
orlydbv1 "next.orly.dev/pkg/proto/orlydb/v1" |
||||
) |
||||
|
||||
// Client implements the database.Database interface via gRPC.
|
||||
type Client struct { |
||||
conn *grpc.ClientConn |
||||
client orlydbv1.DatabaseServiceClient |
||||
ready chan struct{} |
||||
path string |
||||
} |
||||
|
||||
// Verify Client implements database.Database at compile time.
|
||||
var _ database.Database = (*Client)(nil) |
||||
|
||||
// ClientConfig holds configuration for the gRPC client.
|
||||
type ClientConfig struct { |
||||
ServerAddress string |
||||
ConnectTimeout time.Duration |
||||
} |
||||
|
||||
// New creates a new gRPC database client.
|
||||
func New(ctx context.Context, cfg *ClientConfig) (*Client, error) { |
||||
timeout := cfg.ConnectTimeout |
||||
if timeout == 0 { |
||||
timeout = 10 * time.Second |
||||
} |
||||
|
||||
dialCtx, cancel := context.WithTimeout(ctx, timeout) |
||||
defer cancel() |
||||
|
||||
conn, err := grpc.DialContext(dialCtx, cfg.ServerAddress, |
||||
grpc.WithTransportCredentials(insecure.NewCredentials()), |
||||
grpc.WithDefaultCallOptions( |
||||
grpc.MaxCallRecvMsgSize(64<<20), // 64MB
|
||||
grpc.MaxCallSendMsgSize(64<<20), // 64MB
|
||||
), |
||||
) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
c := &Client{ |
||||
conn: conn, |
||||
client: orlydbv1.NewDatabaseServiceClient(conn), |
||||
ready: make(chan struct{}), |
||||
path: "grpc://" + cfg.ServerAddress, |
||||
} |
||||
|
||||
// Check if server is ready
|
||||
go c.waitForReady(ctx) |
||||
|
||||
return c, nil |
||||
} |
||||
|
||||
func (c *Client) waitForReady(ctx context.Context) { |
||||
for { |
||||
select { |
||||
case <-ctx.Done(): |
||||
return |
||||
default: |
||||
resp, err := c.client.Ready(ctx, &orlydbv1.Empty{}) |
||||
if err == nil && resp.Ready { |
||||
close(c.ready) |
||||
log.I.F("gRPC database client connected and ready") |
||||
return |
||||
} |
||||
time.Sleep(100 * time.Millisecond) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// === Lifecycle Methods ===
|
||||
|
||||
func (c *Client) Path() string { |
||||
return c.path |
||||
} |
||||
|
||||
func (c *Client) Init(path string) error { |
||||
// Not applicable for remote database
|
||||
return nil |
||||
} |
||||
|
||||
func (c *Client) Sync() error { |
||||
_, err := c.client.Sync(context.Background(), &orlydbv1.Empty{}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) Close() error { |
||||
return c.conn.Close() |
||||
} |
||||
|
||||
func (c *Client) Wipe() error { |
||||
// Not implemented for remote database (dangerous operation)
|
||||
return nil |
||||
} |
||||
|
||||
func (c *Client) SetLogLevel(level string) { |
||||
_, _ = c.client.SetLogLevel(context.Background(), &orlydbv1.SetLogLevelRequest{Level: level}) |
||||
} |
||||
|
||||
func (c *Client) Ready() <-chan struct{} { |
||||
return c.ready |
||||
} |
||||
|
||||
// === Event Storage ===
|
||||
|
||||
func (c *Client) SaveEvent(ctx context.Context, ev *event.E) (exists bool, err error) { |
||||
resp, err := c.client.SaveEvent(ctx, &orlydbv1.SaveEventRequest{ |
||||
Event: orlydbv1.EventToProto(ev), |
||||
}) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
return resp.Exists, nil |
||||
} |
||||
|
||||
func (c *Client) GetSerialsFromFilter(f *filter.F) (serials indextypes.Uint40s, err error) { |
||||
resp, err := c.client.GetSerialsFromFilter(context.Background(), &orlydbv1.GetSerialsFromFilterRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return protoToUint40s(resp), nil |
||||
} |
||||
|
||||
func (c *Client) WouldReplaceEvent(ev *event.E) (bool, indextypes.Uint40s, error) { |
||||
resp, err := c.client.WouldReplaceEvent(context.Background(), &orlydbv1.WouldReplaceEventRequest{ |
||||
Event: orlydbv1.EventToProto(ev), |
||||
}) |
||||
if err != nil { |
||||
return false, nil, err |
||||
} |
||||
serials := make(indextypes.Uint40s, 0, len(resp.ReplacedSerials)) |
||||
for _, s := range resp.ReplacedSerials { |
||||
u := &indextypes.Uint40{} |
||||
_ = u.Set(s) |
||||
serials = append(serials, u) |
||||
} |
||||
return resp.WouldReplace, serials, nil |
||||
} |
||||
|
||||
// === Event Queries ===
|
||||
|
||||
func (c *Client) QueryEvents(ctx context.Context, f *filter.F) (evs event.S, err error) { |
||||
stream, err := c.client.QueryEvents(ctx, &orlydbv1.QueryEventsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return c.collectStreamedEvents(stream) |
||||
} |
||||
|
||||
func (c *Client) QueryAllVersions(ctx context.Context, f *filter.F) (evs event.S, err error) { |
||||
stream, err := c.client.QueryAllVersions(ctx, &orlydbv1.QueryEventsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return c.collectStreamedEvents(stream) |
||||
} |
||||
|
||||
func (c *Client) QueryEventsWithOptions(ctx context.Context, f *filter.F, includeDeleteEvents bool, showAllVersions bool) (evs event.S, err error) { |
||||
stream, err := c.client.QueryEventsWithOptions(ctx, &orlydbv1.QueryEventsWithOptionsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
IncludeDeleteEvents: includeDeleteEvents, |
||||
ShowAllVersions: showAllVersions, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return c.collectStreamedEvents(stream) |
||||
} |
||||
|
||||
func (c *Client) QueryDeleteEventsByTargetId(ctx context.Context, targetEventId []byte) (evs event.S, err error) { |
||||
stream, err := c.client.QueryDeleteEventsByTargetId(ctx, &orlydbv1.QueryDeleteEventsByTargetIdRequest{ |
||||
TargetEventId: targetEventId, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return c.collectStreamedEvents(stream) |
||||
} |
||||
|
||||
func (c *Client) QueryForSerials(ctx context.Context, f *filter.F) (serials indextypes.Uint40s, err error) { |
||||
resp, err := c.client.QueryForSerials(ctx, &orlydbv1.QueryEventsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return protoToUint40s(resp), nil |
||||
} |
||||
|
||||
func (c *Client) QueryForIds(ctx context.Context, f *filter.F) (idPkTs []*store.IdPkTs, err error) { |
||||
resp, err := c.client.QueryForIds(ctx, &orlydbv1.QueryEventsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToIdPkTsList(resp), nil |
||||
} |
||||
|
||||
func (c *Client) CountEvents(ctx context.Context, f *filter.F) (count int, approximate bool, err error) { |
||||
resp, err := c.client.CountEvents(ctx, &orlydbv1.QueryEventsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return 0, false, err |
||||
} |
||||
return int(resp.Count), resp.Approximate, nil |
||||
} |
||||
|
||||
// === Event Retrieval by Serial ===
|
||||
|
||||
func (c *Client) FetchEventBySerial(ser *indextypes.Uint40) (ev *event.E, err error) { |
||||
resp, err := c.client.FetchEventBySerial(context.Background(), &orlydbv1.FetchEventBySerialRequest{ |
||||
Serial: ser.Get(), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if !resp.Found { |
||||
return nil, nil |
||||
} |
||||
return orlydbv1.ProtoToEvent(resp.Event), nil |
||||
} |
||||
|
||||
func (c *Client) FetchEventsBySerials(serials []*indextypes.Uint40) (events map[uint64]*event.E, err error) { |
||||
serialList := make([]uint64, 0, len(serials)) |
||||
for _, s := range serials { |
||||
serialList = append(serialList, s.Get()) |
||||
} |
||||
resp, err := c.client.FetchEventsBySerials(context.Background(), &orlydbv1.FetchEventsBySerialRequest{ |
||||
Serials: serialList, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToEventMap(resp), nil |
||||
} |
||||
|
||||
func (c *Client) GetSerialById(id []byte) (ser *indextypes.Uint40, err error) { |
||||
resp, err := c.client.GetSerialById(context.Background(), &orlydbv1.GetSerialByIdRequest{ |
||||
Id: id, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if !resp.Found { |
||||
return nil, nil |
||||
} |
||||
u := &indextypes.Uint40{} |
||||
_ = u.Set(resp.Serial) |
||||
return u, nil |
||||
} |
||||
|
||||
func (c *Client) GetSerialsByIds(ids *tag.T) (serials map[string]*indextypes.Uint40, err error) { |
||||
idList := make([][]byte, 0, len(ids.T)) |
||||
for _, id := range ids.T { |
||||
idList = append(idList, id) |
||||
} |
||||
resp, err := c.client.GetSerialsByIds(context.Background(), &orlydbv1.GetSerialsByIdsRequest{ |
||||
Ids: idList, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
result := make(map[string]*indextypes.Uint40) |
||||
for k, v := range resp.Serials { |
||||
u := &indextypes.Uint40{} |
||||
_ = u.Set(v) |
||||
result[k] = u |
||||
} |
||||
return result, nil |
||||
} |
||||
|
||||
func (c *Client) GetSerialsByIdsWithFilter(ids *tag.T, fn func(ev *event.E, ser *indextypes.Uint40) bool) (serials map[string]*indextypes.Uint40, err error) { |
||||
// Note: Filter function cannot be passed over gRPC, so we just get all serials
|
||||
return c.GetSerialsByIds(ids) |
||||
} |
||||
|
||||
func (c *Client) GetSerialsByRange(idx database.Range) (serials indextypes.Uint40s, err error) { |
||||
resp, err := c.client.GetSerialsByRange(context.Background(), &orlydbv1.GetSerialsByRangeRequest{ |
||||
Range: orlydbv1.RangeToProto(idx), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return protoToUint40s(resp), nil |
||||
} |
||||
|
||||
func (c *Client) GetFullIdPubkeyBySerial(ser *indextypes.Uint40) (fidpk *store.IdPkTs, err error) { |
||||
resp, err := c.client.GetFullIdPubkeyBySerial(context.Background(), &orlydbv1.GetFullIdPubkeyBySerialRequest{ |
||||
Serial: ser.Get(), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToIdPkTs(resp), nil |
||||
} |
||||
|
||||
func (c *Client) GetFullIdPubkeyBySerials(sers []*indextypes.Uint40) (fidpks []*store.IdPkTs, err error) { |
||||
serialList := make([]uint64, 0, len(sers)) |
||||
for _, s := range sers { |
||||
serialList = append(serialList, s.Get()) |
||||
} |
||||
resp, err := c.client.GetFullIdPubkeyBySerials(context.Background(), &orlydbv1.GetFullIdPubkeyBySerialsRequest{ |
||||
Serials: serialList, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToIdPkTsList(resp), nil |
||||
} |
||||
|
||||
// === Event Deletion ===
|
||||
|
||||
func (c *Client) DeleteEvent(ctx context.Context, eid []byte) error { |
||||
_, err := c.client.DeleteEvent(ctx, &orlydbv1.DeleteEventRequest{ |
||||
EventId: eid, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) DeleteEventBySerial(ctx context.Context, ser *indextypes.Uint40, ev *event.E) error { |
||||
_, err := c.client.DeleteEventBySerial(ctx, &orlydbv1.DeleteEventBySerialRequest{ |
||||
Serial: ser.Get(), |
||||
Event: orlydbv1.EventToProto(ev), |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) DeleteExpired() { |
||||
_, _ = c.client.DeleteExpired(context.Background(), &orlydbv1.Empty{}) |
||||
} |
||||
|
||||
func (c *Client) ProcessDelete(ev *event.E, admins [][]byte) error { |
||||
_, err := c.client.ProcessDelete(context.Background(), &orlydbv1.ProcessDeleteRequest{ |
||||
Event: orlydbv1.EventToProto(ev), |
||||
Admins: admins, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) CheckForDeleted(ev *event.E, admins [][]byte) error { |
||||
_, err := c.client.CheckForDeleted(context.Background(), &orlydbv1.CheckForDeletedRequest{ |
||||
Event: orlydbv1.EventToProto(ev), |
||||
Admins: admins, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
// === Import/Export ===
|
||||
|
||||
func (c *Client) Import(rr io.Reader) { |
||||
stream, err := c.client.Import(context.Background()) |
||||
if chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
buf := make([]byte, 64*1024) |
||||
for { |
||||
n, err := rr.Read(buf) |
||||
if err == io.EOF { |
||||
break |
||||
} |
||||
if chk.E(err) { |
||||
return |
||||
} |
||||
if err := stream.Send(&orlydbv1.ImportChunk{Data: buf[:n]}); chk.E(err) { |
||||
return |
||||
} |
||||
} |
||||
|
||||
_, _ = stream.CloseAndRecv() |
||||
} |
||||
|
||||
func (c *Client) Export(ctx context.Context, w io.Writer, pubkeys ...[]byte) { |
||||
stream, err := c.client.Export(ctx, &orlydbv1.ExportRequest{ |
||||
Pubkeys: pubkeys, |
||||
}) |
||||
if chk.E(err) { |
||||
return |
||||
} |
||||
|
||||
for { |
||||
chunk, err := stream.Recv() |
||||
if err == io.EOF { |
||||
return |
||||
} |
||||
if chk.E(err) { |
||||
return |
||||
} |
||||
if _, err := w.Write(chunk.Data); chk.E(err) { |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (c *Client) ImportEventsFromReader(ctx context.Context, rr io.Reader) error { |
||||
c.Import(rr) |
||||
return nil |
||||
} |
||||
|
||||
func (c *Client) ImportEventsFromStrings(ctx context.Context, eventJSONs []string, policyManager interface{ CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) }) error { |
||||
_, err := c.client.ImportEventsFromStrings(ctx, &orlydbv1.ImportEventsFromStringsRequest{ |
||||
EventJsons: eventJSONs, |
||||
CheckPolicy: policyManager != nil, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
// === Relay Identity ===
|
||||
|
||||
func (c *Client) GetRelayIdentitySecret() (skb []byte, err error) { |
||||
resp, err := c.client.GetRelayIdentitySecret(context.Background(), &orlydbv1.Empty{}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return resp.SecretKey, nil |
||||
} |
||||
|
||||
func (c *Client) SetRelayIdentitySecret(skb []byte) error { |
||||
_, err := c.client.SetRelayIdentitySecret(context.Background(), &orlydbv1.SetRelayIdentitySecretRequest{ |
||||
SecretKey: skb, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) GetOrCreateRelayIdentitySecret() (skb []byte, err error) { |
||||
resp, err := c.client.GetOrCreateRelayIdentitySecret(context.Background(), &orlydbv1.Empty{}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return resp.SecretKey, nil |
||||
} |
||||
|
||||
// === Markers ===
|
||||
|
||||
func (c *Client) SetMarker(key string, value []byte) error { |
||||
_, err := c.client.SetMarker(context.Background(), &orlydbv1.SetMarkerRequest{ |
||||
Key: key, |
||||
Value: value, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) GetMarker(key string) (value []byte, err error) { |
||||
resp, err := c.client.GetMarker(context.Background(), &orlydbv1.GetMarkerRequest{ |
||||
Key: key, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if !resp.Found { |
||||
return nil, nil |
||||
} |
||||
return resp.Value, nil |
||||
} |
||||
|
||||
func (c *Client) HasMarker(key string) bool { |
||||
resp, err := c.client.HasMarker(context.Background(), &orlydbv1.HasMarkerRequest{ |
||||
Key: key, |
||||
}) |
||||
if err != nil { |
||||
return false |
||||
} |
||||
return resp.Exists |
||||
} |
||||
|
||||
func (c *Client) DeleteMarker(key string) error { |
||||
_, err := c.client.DeleteMarker(context.Background(), &orlydbv1.DeleteMarkerRequest{ |
||||
Key: key, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
// === Subscriptions ===
|
||||
|
||||
func (c *Client) GetSubscription(pubkey []byte) (*database.Subscription, error) { |
||||
resp, err := c.client.GetSubscription(context.Background(), &orlydbv1.GetSubscriptionRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToSubscription(resp), nil |
||||
} |
||||
|
||||
func (c *Client) IsSubscriptionActive(pubkey []byte) (bool, error) { |
||||
resp, err := c.client.IsSubscriptionActive(context.Background(), &orlydbv1.IsSubscriptionActiveRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
return resp.Active, nil |
||||
} |
||||
|
||||
func (c *Client) ExtendSubscription(pubkey []byte, days int) error { |
||||
_, err := c.client.ExtendSubscription(context.Background(), &orlydbv1.ExtendSubscriptionRequest{ |
||||
Pubkey: pubkey, |
||||
Days: int32(days), |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) RecordPayment(pubkey []byte, amount int64, invoice, preimage string) error { |
||||
_, err := c.client.RecordPayment(context.Background(), &orlydbv1.RecordPaymentRequest{ |
||||
Pubkey: pubkey, |
||||
Amount: amount, |
||||
Invoice: invoice, |
||||
Preimage: preimage, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) GetPaymentHistory(pubkey []byte) ([]database.Payment, error) { |
||||
resp, err := c.client.GetPaymentHistory(context.Background(), &orlydbv1.GetPaymentHistoryRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToPaymentList(resp), nil |
||||
} |
||||
|
||||
func (c *Client) ExtendBlossomSubscription(pubkey []byte, tier string, storageMB int64, daysExtended int) error { |
||||
_, err := c.client.ExtendBlossomSubscription(context.Background(), &orlydbv1.ExtendBlossomSubscriptionRequest{ |
||||
Pubkey: pubkey, |
||||
Tier: tier, |
||||
StorageMb: storageMB, |
||||
DaysExtended: int32(daysExtended), |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) GetBlossomStorageQuota(pubkey []byte) (quotaMB int64, err error) { |
||||
resp, err := c.client.GetBlossomStorageQuota(context.Background(), &orlydbv1.GetBlossomStorageQuotaRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return resp.QuotaMb, nil |
||||
} |
||||
|
||||
func (c *Client) IsFirstTimeUser(pubkey []byte) (bool, error) { |
||||
resp, err := c.client.IsFirstTimeUser(context.Background(), &orlydbv1.IsFirstTimeUserRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
return resp.FirstTime, nil |
||||
} |
||||
|
||||
// === NIP-43 ===
|
||||
|
||||
func (c *Client) AddNIP43Member(pubkey []byte, inviteCode string) error { |
||||
_, err := c.client.AddNIP43Member(context.Background(), &orlydbv1.AddNIP43MemberRequest{ |
||||
Pubkey: pubkey, |
||||
InviteCode: inviteCode, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) RemoveNIP43Member(pubkey []byte) error { |
||||
_, err := c.client.RemoveNIP43Member(context.Background(), &orlydbv1.RemoveNIP43MemberRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) IsNIP43Member(pubkey []byte) (isMember bool, err error) { |
||||
resp, err := c.client.IsNIP43Member(context.Background(), &orlydbv1.IsNIP43MemberRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
return resp.IsMember, nil |
||||
} |
||||
|
||||
func (c *Client) GetNIP43Membership(pubkey []byte) (*database.NIP43Membership, error) { |
||||
resp, err := c.client.GetNIP43Membership(context.Background(), &orlydbv1.GetNIP43MembershipRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToNIP43Membership(resp), nil |
||||
} |
||||
|
||||
func (c *Client) GetAllNIP43Members() ([][]byte, error) { |
||||
resp, err := c.client.GetAllNIP43Members(context.Background(), &orlydbv1.Empty{}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return resp.Pubkeys, nil |
||||
} |
||||
|
||||
func (c *Client) StoreInviteCode(code string, expiresAt time.Time) error { |
||||
_, err := c.client.StoreInviteCode(context.Background(), &orlydbv1.StoreInviteCodeRequest{ |
||||
Code: code, |
||||
ExpiresAt: expiresAt.Unix(), |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) ValidateInviteCode(code string) (valid bool, err error) { |
||||
resp, err := c.client.ValidateInviteCode(context.Background(), &orlydbv1.ValidateInviteCodeRequest{ |
||||
Code: code, |
||||
}) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
return resp.Valid, nil |
||||
} |
||||
|
||||
func (c *Client) DeleteInviteCode(code string) error { |
||||
_, err := c.client.DeleteInviteCode(context.Background(), &orlydbv1.DeleteInviteCodeRequest{ |
||||
Code: code, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) PublishNIP43MembershipEvent(kind int, pubkey []byte) error { |
||||
_, err := c.client.PublishNIP43MembershipEvent(context.Background(), &orlydbv1.PublishNIP43MembershipEventRequest{ |
||||
Kind: int32(kind), |
||||
Pubkey: pubkey, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
// === Migrations ===
|
||||
|
||||
func (c *Client) RunMigrations() { |
||||
_, _ = c.client.RunMigrations(context.Background(), &orlydbv1.Empty{}) |
||||
} |
||||
|
||||
// === Query Cache ===
|
||||
|
||||
func (c *Client) GetCachedJSON(f *filter.F) ([][]byte, bool) { |
||||
resp, err := c.client.GetCachedJSON(context.Background(), &orlydbv1.GetCachedJSONRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return nil, false |
||||
} |
||||
return resp.JsonItems, resp.Found |
||||
} |
||||
|
||||
func (c *Client) CacheMarshaledJSON(f *filter.F, marshaledJSON [][]byte) { |
||||
_, _ = c.client.CacheMarshaledJSON(context.Background(), &orlydbv1.CacheMarshaledJSONRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
JsonItems: marshaledJSON, |
||||
}) |
||||
} |
||||
|
||||
func (c *Client) GetCachedEvents(f *filter.F) (event.S, bool) { |
||||
resp, err := c.client.GetCachedEvents(context.Background(), &orlydbv1.GetCachedEventsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
}) |
||||
if err != nil { |
||||
return nil, false |
||||
} |
||||
return orlydbv1.ProtoToEvents(resp.Events), resp.Found |
||||
} |
||||
|
||||
func (c *Client) CacheEvents(f *filter.F, events event.S) { |
||||
_, _ = c.client.CacheEvents(context.Background(), &orlydbv1.CacheEventsRequest{ |
||||
Filter: orlydbv1.FilterToProto(f), |
||||
Events: orlydbv1.EventsToProto(events), |
||||
}) |
||||
} |
||||
|
||||
func (c *Client) InvalidateQueryCache() { |
||||
_, _ = c.client.InvalidateQueryCache(context.Background(), &orlydbv1.Empty{}) |
||||
} |
||||
|
||||
// === Access Tracking ===
|
||||
|
||||
func (c *Client) RecordEventAccess(serial uint64, connectionID string) error { |
||||
_, err := c.client.RecordEventAccess(context.Background(), &orlydbv1.RecordEventAccessRequest{ |
||||
Serial: serial, |
||||
ConnectionId: connectionID, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) GetEventAccessInfo(serial uint64) (lastAccess int64, accessCount uint32, err error) { |
||||
resp, err := c.client.GetEventAccessInfo(context.Background(), &orlydbv1.GetEventAccessInfoRequest{ |
||||
Serial: serial, |
||||
}) |
||||
if err != nil { |
||||
return 0, 0, err |
||||
} |
||||
return resp.LastAccess, resp.AccessCount, nil |
||||
} |
||||
|
||||
func (c *Client) GetLeastAccessedEvents(limit int, minAgeSec int64) (serials []uint64, err error) { |
||||
resp, err := c.client.GetLeastAccessedEvents(context.Background(), &orlydbv1.GetLeastAccessedEventsRequest{ |
||||
Limit: int32(limit), |
||||
MinAgeSec: minAgeSec, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return resp.Serials, nil |
||||
} |
||||
|
||||
// === Blob Storage (Blossom) ===
|
||||
|
||||
func (c *Client) SaveBlob(sha256Hash []byte, data []byte, pubkey []byte, mimeType string, extension string) error { |
||||
_, err := c.client.SaveBlob(context.Background(), &orlydbv1.SaveBlobRequest{ |
||||
Sha256Hash: sha256Hash, |
||||
Data: data, |
||||
Pubkey: pubkey, |
||||
MimeType: mimeType, |
||||
Extension: extension, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) GetBlob(sha256Hash []byte) (data []byte, metadata *database.BlobMetadata, err error) { |
||||
resp, err := c.client.GetBlob(context.Background(), &orlydbv1.GetBlobRequest{ |
||||
Sha256Hash: sha256Hash, |
||||
}) |
||||
if err != nil { |
||||
return nil, nil, err |
||||
} |
||||
if !resp.Found { |
||||
return nil, nil, nil |
||||
} |
||||
return resp.Data, orlydbv1.ProtoToBlobMetadata(resp.Metadata), nil |
||||
} |
||||
|
||||
func (c *Client) HasBlob(sha256Hash []byte) (exists bool, err error) { |
||||
resp, err := c.client.HasBlob(context.Background(), &orlydbv1.HasBlobRequest{ |
||||
Sha256Hash: sha256Hash, |
||||
}) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
return resp.Exists, nil |
||||
} |
||||
|
||||
func (c *Client) DeleteBlob(sha256Hash []byte, pubkey []byte) error { |
||||
_, err := c.client.DeleteBlob(context.Background(), &orlydbv1.DeleteBlobRequest{ |
||||
Sha256Hash: sha256Hash, |
||||
Pubkey: pubkey, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) ListBlobs(pubkey []byte, since, until int64) ([]*database.BlobDescriptor, error) { |
||||
resp, err := c.client.ListBlobs(context.Background(), &orlydbv1.ListBlobsRequest{ |
||||
Pubkey: pubkey, |
||||
Since: since, |
||||
Until: until, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToBlobDescriptorList(resp.Descriptors), nil |
||||
} |
||||
|
||||
func (c *Client) GetBlobMetadata(sha256Hash []byte) (*database.BlobMetadata, error) { |
||||
resp, err := c.client.GetBlobMetadata(context.Background(), &orlydbv1.GetBlobMetadataRequest{ |
||||
Sha256Hash: sha256Hash, |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToBlobMetadata(resp), nil |
||||
} |
||||
|
||||
func (c *Client) GetTotalBlobStorageUsed(pubkey []byte) (totalMB int64, err error) { |
||||
resp, err := c.client.GetTotalBlobStorageUsed(context.Background(), &orlydbv1.GetTotalBlobStorageUsedRequest{ |
||||
Pubkey: pubkey, |
||||
}) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return resp.TotalMb, nil |
||||
} |
||||
|
||||
func (c *Client) SaveBlobReport(sha256Hash []byte, reportData []byte) error { |
||||
_, err := c.client.SaveBlobReport(context.Background(), &orlydbv1.SaveBlobReportRequest{ |
||||
Sha256Hash: sha256Hash, |
||||
ReportData: reportData, |
||||
}) |
||||
return err |
||||
} |
||||
|
||||
func (c *Client) ListAllBlobUserStats() ([]*database.UserBlobStats, error) { |
||||
resp, err := c.client.ListAllBlobUserStats(context.Background(), &orlydbv1.Empty{}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return orlydbv1.ProtoToUserBlobStatsList(resp.Stats), nil |
||||
} |
||||
|
||||
// === Utility ===
|
||||
|
||||
func (c *Client) EventIdsBySerial(start uint64, count int) (evs []uint64, err error) { |
||||
resp, err := c.client.EventIdsBySerial(context.Background(), &orlydbv1.EventIdsBySerialRequest{ |
||||
Start: start, |
||||
Count: int32(count), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return resp.EventIds, nil |
||||
} |
||||
|
||||
// === Helper Methods ===
|
||||
|
||||
type eventStream interface { |
||||
Recv() (*orlydbv1.EventBatch, error) |
||||
} |
||||
|
||||
func (c *Client) collectStreamedEvents(stream eventStream) (event.S, error) { |
||||
var result event.S |
||||
for { |
||||
batch, err := stream.Recv() |
||||
if err == io.EOF { |
||||
return result, nil |
||||
} |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
for _, ev := range batch.Events { |
||||
result = append(result, orlydbv1.ProtoToEvent(ev)) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func protoToUint40s(resp *orlydbv1.SerialList) indextypes.Uint40s { |
||||
if resp == nil { |
||||
return nil |
||||
} |
||||
result := make(indextypes.Uint40s, 0, len(resp.Serials)) |
||||
for _, s := range resp.Serials { |
||||
u := &indextypes.Uint40{} |
||||
_ = u.Set(s) |
||||
result = append(result, u) |
||||
} |
||||
return result |
||||
} |
||||
@ -0,0 +1,20 @@
@@ -0,0 +1,20 @@
|
||||
package grpc |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"next.orly.dev/pkg/database" |
||||
) |
||||
|
||||
func init() { |
||||
database.RegisterGRPCFactory(NewFromConfig) |
||||
} |
||||
|
||||
// NewFromConfig creates a new gRPC database client from the database config.
|
||||
func NewFromConfig(ctx context.Context, cancel context.CancelFunc, cfg *database.DatabaseConfig) (database.Database, error) { |
||||
clientCfg := &ClientConfig{ |
||||
ServerAddress: cfg.GRPCServerAddress, |
||||
ConnectTimeout: cfg.GRPCConnectTimeout, |
||||
} |
||||
return New(ctx, clientCfg) |
||||
} |
||||
@ -0,0 +1,57 @@
@@ -0,0 +1,57 @@
|
||||
package neo4j |
||||
|
||||
import ( |
||||
"errors" |
||||
|
||||
"next.orly.dev/pkg/database" |
||||
) |
||||
|
||||
// Blob storage methods - Neo4j does not support blob storage
|
||||
// These are stub implementations that return errors
|
||||
|
||||
var errBlobNotSupported = errors.New("blob storage not supported in Neo4j backend") |
||||
|
||||
// SaveBlob stores a blob with its metadata (not supported in Neo4j)
|
||||
func (n *N) SaveBlob(sha256Hash []byte, data []byte, pubkey []byte, mimeType string, extension string) error { |
||||
return errBlobNotSupported |
||||
} |
||||
|
||||
// GetBlob retrieves blob data by SHA256 hash (not supported in Neo4j)
|
||||
func (n *N) GetBlob(sha256Hash []byte) (data []byte, metadata *database.BlobMetadata, err error) { |
||||
return nil, nil, errBlobNotSupported |
||||
} |
||||
|
||||
// HasBlob checks if a blob exists (not supported in Neo4j)
|
||||
func (n *N) HasBlob(sha256Hash []byte) (exists bool, err error) { |
||||
return false, errBlobNotSupported |
||||
} |
||||
|
||||
// DeleteBlob deletes a blob and its metadata (not supported in Neo4j)
|
||||
func (n *N) DeleteBlob(sha256Hash []byte, pubkey []byte) error { |
||||
return errBlobNotSupported |
||||
} |
||||
|
||||
// ListBlobs lists all blobs for a given pubkey (not supported in Neo4j)
|
||||
func (n *N) ListBlobs(pubkey []byte, since, until int64) ([]*database.BlobDescriptor, error) { |
||||
return nil, errBlobNotSupported |
||||
} |
||||
|
||||
// GetBlobMetadata retrieves only metadata for a blob (not supported in Neo4j)
|
||||
func (n *N) GetBlobMetadata(sha256Hash []byte) (*database.BlobMetadata, error) { |
||||
return nil, errBlobNotSupported |
||||
} |
||||
|
||||
// GetTotalBlobStorageUsed calculates total storage used by a pubkey in MB (not supported in Neo4j)
|
||||
func (n *N) GetTotalBlobStorageUsed(pubkey []byte) (totalMB int64, err error) { |
||||
return 0, errBlobNotSupported |
||||
} |
||||
|
||||
// SaveBlobReport stores a report for a blob (not supported in Neo4j)
|
||||
func (n *N) SaveBlobReport(sha256Hash []byte, reportData []byte) error { |
||||
return errBlobNotSupported |
||||
} |
||||
|
||||
// ListAllBlobUserStats returns storage statistics for all users (not supported in Neo4j)
|
||||
func (n *N) ListAllBlobUserStats() ([]*database.UserBlobStats, error) { |
||||
return nil, errBlobNotSupported |
||||
} |
||||
@ -0,0 +1,619 @@
@@ -0,0 +1,619 @@
|
||||
// Package orlydbv1 provides type converters between proto messages and Go types.
|
||||
package orlydbv1 |
||||
|
||||
import ( |
||||
"time" |
||||
|
||||
"git.mleku.dev/mleku/nostr/encoders/event" |
||||
"git.mleku.dev/mleku/nostr/encoders/filter" |
||||
"git.mleku.dev/mleku/nostr/encoders/kind" |
||||
"git.mleku.dev/mleku/nostr/encoders/tag" |
||||
"git.mleku.dev/mleku/nostr/encoders/timestamp" |
||||
"next.orly.dev/pkg/database" |
||||
indextypes "next.orly.dev/pkg/database/indexes/types" |
||||
"next.orly.dev/pkg/interfaces/store" |
||||
) |
||||
|
||||
// EventToProto converts a nostr event.E to a proto Event.
|
||||
// Binary fields (ID, Pubkey, Sig) are copied directly.
|
||||
// Tags are converted preserving binary values for e/p tags.
|
||||
func EventToProto(ev *event.E) *Event { |
||||
if ev == nil { |
||||
return nil |
||||
} |
||||
|
||||
pb := &Event{ |
||||
Id: ev.ID, |
||||
Pubkey: ev.Pubkey, |
||||
CreatedAt: ev.CreatedAt, |
||||
Kind: uint32(ev.Kind), |
||||
Content: ev.Content, |
||||
Sig: ev.Sig, |
||||
} |
||||
|
||||
if ev.Tags != nil { |
||||
pb.Tags = make([]*Tag, 0, len(*ev.Tags)) |
||||
for _, t := range *ev.Tags { |
||||
pbTag := &Tag{ |
||||
Values: make([][]byte, 0, len(t.T)), |
||||
} |
||||
for _, v := range t.T { |
||||
// Copy bytes directly to preserve binary data
|
||||
val := make([]byte, len(v)) |
||||
copy(val, v) |
||||
pbTag.Values = append(pbTag.Values, val) |
||||
} |
||||
pb.Tags = append(pb.Tags, pbTag) |
||||
} |
||||
} |
||||
|
||||
return pb |
||||
} |
||||
|
||||
// ProtoToEvent converts a proto Event to a nostr event.E.
|
||||
func ProtoToEvent(pb *Event) *event.E { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
|
||||
ev := event.New() |
||||
ev.ID = pb.Id |
||||
ev.Pubkey = pb.Pubkey |
||||
ev.CreatedAt = pb.CreatedAt |
||||
ev.Kind = uint16(pb.Kind) |
||||
ev.Content = pb.Content |
||||
ev.Sig = pb.Sig |
||||
|
||||
if len(pb.Tags) > 0 { |
||||
tags := tag.NewSWithCap(len(pb.Tags)) |
||||
for _, pbTag := range pb.Tags { |
||||
t := tag.NewWithCap(len(pbTag.Values)) |
||||
for _, v := range pbTag.Values { |
||||
t.T = append(t.T, v) |
||||
} |
||||
*tags = append(*tags, t) |
||||
} |
||||
ev.Tags = tags |
||||
} |
||||
|
||||
return ev |
||||
} |
||||
|
||||
// FilterToProto converts a nostr filter.F to a proto Filter.
|
||||
func FilterToProto(f *filter.F) *Filter { |
||||
if f == nil { |
||||
return nil |
||||
} |
||||
|
||||
pb := &Filter{} |
||||
|
||||
// IDs
|
||||
if f.Ids != nil && len(f.Ids.T) > 0 { |
||||
pb.Ids = make([][]byte, 0, len(f.Ids.T)) |
||||
for _, id := range f.Ids.T { |
||||
pb.Ids = append(pb.Ids, id) |
||||
} |
||||
} |
||||
|
||||
// Kinds
|
||||
if f.Kinds != nil && f.Kinds.Len() > 0 { |
||||
pb.Kinds = make([]uint32, 0, f.Kinds.Len()) |
||||
for _, k := range f.Kinds.K { |
||||
pb.Kinds = append(pb.Kinds, uint32(k.K)) |
||||
} |
||||
} |
||||
|
||||
// Authors
|
||||
if f.Authors != nil && len(f.Authors.T) > 0 { |
||||
pb.Authors = make([][]byte, 0, len(f.Authors.T)) |
||||
for _, a := range f.Authors.T { |
||||
pb.Authors = append(pb.Authors, a) |
||||
} |
||||
} |
||||
|
||||
// Tags (e.g., #e, #p, #t)
|
||||
if f.Tags != nil && len(*f.Tags) > 0 { |
||||
pb.Tags = make(map[string]*TagSet) |
||||
for _, t := range *f.Tags { |
||||
if len(t.T) >= 2 { |
||||
key := string(t.T[0]) |
||||
ts, exists := pb.Tags[key] |
||||
if !exists { |
||||
ts = &TagSet{} |
||||
pb.Tags[key] = ts |
||||
} |
||||
// Add all values after the key
|
||||
for _, v := range t.T[1:] { |
||||
ts.Values = append(ts.Values, v) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Since
|
||||
if f.Since != nil { |
||||
since := f.Since.I64() |
||||
pb.Since = &since |
||||
} |
||||
|
||||
// Until
|
||||
if f.Until != nil { |
||||
until := f.Until.I64() |
||||
pb.Until = &until |
||||
} |
||||
|
||||
// Search
|
||||
if len(f.Search) > 0 { |
||||
pb.Search = f.Search |
||||
} |
||||
|
||||
// Limit
|
||||
if f.Limit != nil { |
||||
limit := uint32(*f.Limit) |
||||
pb.Limit = &limit |
||||
} |
||||
|
||||
return pb |
||||
} |
||||
|
||||
// ProtoToFilter converts a proto Filter to a nostr filter.F.
|
||||
func ProtoToFilter(pb *Filter) *filter.F { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
|
||||
f := filter.New() |
||||
|
||||
// IDs
|
||||
if len(pb.Ids) > 0 { |
||||
f.Ids = tag.NewWithCap(len(pb.Ids)) |
||||
for _, id := range pb.Ids { |
||||
f.Ids.T = append(f.Ids.T, id) |
||||
} |
||||
} |
||||
|
||||
// Kinds
|
||||
if len(pb.Kinds) > 0 { |
||||
kinds := kind.NewWithCap(len(pb.Kinds)) |
||||
for _, k := range pb.Kinds { |
||||
kinds.K = append(kinds.K, kind.New(uint16(k))) |
||||
} |
||||
f.Kinds = kinds |
||||
} |
||||
|
||||
// Authors
|
||||
if len(pb.Authors) > 0 { |
||||
f.Authors = tag.NewWithCap(len(pb.Authors)) |
||||
for _, a := range pb.Authors { |
||||
f.Authors.T = append(f.Authors.T, a) |
||||
} |
||||
} |
||||
|
||||
// Tags
|
||||
if len(pb.Tags) > 0 { |
||||
tags := tag.NewSWithCap(len(pb.Tags)) |
||||
for key, ts := range pb.Tags { |
||||
for _, v := range ts.Values { |
||||
t := tag.NewWithCap(2) |
||||
t.T = append(t.T, []byte(key), v) |
||||
*tags = append(*tags, t) |
||||
} |
||||
} |
||||
f.Tags = tags |
||||
} |
||||
|
||||
// Since
|
||||
if pb.Since != nil { |
||||
f.Since = timestamp.FromUnix(*pb.Since) |
||||
} |
||||
|
||||
// Until
|
||||
if pb.Until != nil { |
||||
f.Until = timestamp.FromUnix(*pb.Until) |
||||
} |
||||
|
||||
// Search
|
||||
if len(pb.Search) > 0 { |
||||
f.Search = pb.Search |
||||
} |
||||
|
||||
// Limit
|
||||
if pb.Limit != nil { |
||||
limit := uint(*pb.Limit) |
||||
f.Limit = &limit |
||||
} |
||||
|
||||
return f |
||||
} |
||||
|
||||
// Uint40ToProto converts a indextypes.Uint40 to a proto Uint40.
|
||||
func Uint40ToProto(u *indextypes.Uint40) *Uint40 { |
||||
if u == nil { |
||||
return nil |
||||
} |
||||
return &Uint40{Value: u.Get()} |
||||
} |
||||
|
||||
// ProtoToUint40 converts a proto Uint40 to a indextypes.Uint40.
|
||||
func ProtoToUint40(pb *Uint40) *indextypes.Uint40 { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
return newUint40(pb.Value) |
||||
} |
||||
|
||||
// Uint40sToProto converts a slice of Uint40s to a SerialList.
|
||||
func Uint40sToProto(serials indextypes.Uint40s) *SerialList { |
||||
result := &SerialList{ |
||||
Serials: make([]uint64, 0, len(serials)), |
||||
} |
||||
for _, s := range serials { |
||||
result.Serials = append(result.Serials, s.Get()) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// ProtoToUint40s converts a SerialList to a slice of Uint40 pointers.
|
||||
func ProtoToUint40s(pb *SerialList) []*indextypes.Uint40 { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
result := make([]*indextypes.Uint40, 0, len(pb.Serials)) |
||||
for _, s := range pb.Serials { |
||||
result = append(result, newUint40(s)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// IdPkTsToProto converts a store.IdPkTs to a proto IdPkTs.
|
||||
func IdPkTsToProto(i *store.IdPkTs) *IdPkTs { |
||||
if i == nil { |
||||
return nil |
||||
} |
||||
return &IdPkTs{ |
||||
Id: i.Id, |
||||
Pubkey: i.Pub, |
||||
Timestamp: i.Ts, |
||||
Serial: i.Ser, |
||||
} |
||||
} |
||||
|
||||
// ProtoToIdPkTs converts a proto IdPkTs to a store.IdPkTs.
|
||||
func ProtoToIdPkTs(pb *IdPkTs) *store.IdPkTs { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
return &store.IdPkTs{ |
||||
Id: pb.Id, |
||||
Pub: pb.Pubkey, |
||||
Ts: pb.Timestamp, |
||||
Ser: pb.Serial, |
||||
} |
||||
} |
||||
|
||||
// IdPkTsListToProto converts a slice of IdPkTs to a proto IdPkTsList.
|
||||
func IdPkTsListToProto(items []*store.IdPkTs) *IdPkTsList { |
||||
result := &IdPkTsList{ |
||||
Items: make([]*IdPkTs, 0, len(items)), |
||||
} |
||||
for _, item := range items { |
||||
result.Items = append(result.Items, IdPkTsToProto(item)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// ProtoToIdPkTsList converts a proto IdPkTsList to a slice of IdPkTs.
|
||||
func ProtoToIdPkTsList(pb *IdPkTsList) []*store.IdPkTs { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
result := make([]*store.IdPkTs, 0, len(pb.Items)) |
||||
for _, item := range pb.Items { |
||||
result = append(result, ProtoToIdPkTs(item)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// SubscriptionToProto converts a database.Subscription to a proto Subscription.
|
||||
func SubscriptionToProto(s *database.Subscription, pubkey []byte) *Subscription { |
||||
if s == nil { |
||||
return nil |
||||
} |
||||
return &Subscription{ |
||||
Pubkey: pubkey, |
||||
TrialEnd: s.TrialEnd.Unix(), |
||||
PaidUntil: s.PaidUntil.Unix(), |
||||
BlossomLevel: s.BlossomLevel, |
||||
BlossomStorageMb: s.BlossomStorage, |
||||
} |
||||
} |
||||
|
||||
// ProtoToSubscription converts a proto Subscription to a database.Subscription.
|
||||
func ProtoToSubscription(pb *Subscription) *database.Subscription { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
return &database.Subscription{ |
||||
TrialEnd: timeFromUnix(pb.TrialEnd), |
||||
PaidUntil: timeFromUnix(pb.PaidUntil), |
||||
BlossomLevel: pb.BlossomLevel, |
||||
BlossomStorage: pb.BlossomStorageMb, |
||||
} |
||||
} |
||||
|
||||
// PaymentToProto converts a database.Payment to a proto Payment.
|
||||
func PaymentToProto(p *database.Payment) *Payment { |
||||
return &Payment{ |
||||
Amount: p.Amount, |
||||
Timestamp: p.Timestamp.Unix(), |
||||
Invoice: p.Invoice, |
||||
Preimage: p.Preimage, |
||||
} |
||||
} |
||||
|
||||
// ProtoToPayment converts a proto Payment to a database.Payment.
|
||||
func ProtoToPayment(pb *Payment) *database.Payment { |
||||
return &database.Payment{ |
||||
Amount: pb.Amount, |
||||
Timestamp: timeFromUnix(pb.Timestamp), |
||||
Invoice: pb.Invoice, |
||||
Preimage: pb.Preimage, |
||||
} |
||||
} |
||||
|
||||
// PaymentListToProto converts a slice of payments to a proto PaymentList.
|
||||
func PaymentListToProto(payments []database.Payment) *PaymentList { |
||||
result := &PaymentList{ |
||||
Payments: make([]*Payment, 0, len(payments)), |
||||
} |
||||
for _, p := range payments { |
||||
result.Payments = append(result.Payments, PaymentToProto(&p)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// ProtoToPaymentList converts a proto PaymentList to a slice of payments.
|
||||
func ProtoToPaymentList(pb *PaymentList) []database.Payment { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
result := make([]database.Payment, 0, len(pb.Payments)) |
||||
for _, p := range pb.Payments { |
||||
result = append(result, *ProtoToPayment(p)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// NIP43MembershipToProto converts a database.NIP43Membership to a proto NIP43Membership.
|
||||
func NIP43MembershipToProto(m *database.NIP43Membership) *NIP43Membership { |
||||
if m == nil { |
||||
return nil |
||||
} |
||||
return &NIP43Membership{ |
||||
Pubkey: m.Pubkey, |
||||
AddedAt: m.AddedAt.Unix(), |
||||
InviteCode: m.InviteCode, |
||||
} |
||||
} |
||||
|
||||
// ProtoToNIP43Membership converts a proto NIP43Membership to a database.NIP43Membership.
|
||||
func ProtoToNIP43Membership(pb *NIP43Membership) *database.NIP43Membership { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
return &database.NIP43Membership{ |
||||
Pubkey: pb.Pubkey, |
||||
AddedAt: timeFromUnix(pb.AddedAt), |
||||
InviteCode: pb.InviteCode, |
||||
} |
||||
} |
||||
|
||||
// RangeToProto converts a database.Range to a proto Range.
|
||||
func RangeToProto(r database.Range) *Range { |
||||
return &Range{ |
||||
Prefix: r.Start, |
||||
Start: 0, |
||||
End: 0, |
||||
} |
||||
} |
||||
|
||||
// ProtoToRange converts a proto Range to a database.Range.
|
||||
func ProtoToRange(pb *Range) database.Range { |
||||
if pb == nil { |
||||
return database.Range{} |
||||
} |
||||
return database.Range{ |
||||
Start: pb.Prefix, |
||||
End: nil, |
||||
} |
||||
} |
||||
|
||||
// EventMapToProto converts a map of serial->event to a proto EventMap.
|
||||
func EventMapToProto(events map[uint64]*event.E) *EventMap { |
||||
result := &EventMap{ |
||||
Events: make(map[uint64]*Event), |
||||
} |
||||
for serial, ev := range events { |
||||
result.Events[serial] = EventToProto(ev) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// ProtoToEventMap converts a proto EventMap to a map of serial->event.
|
||||
func ProtoToEventMap(pb *EventMap) map[uint64]*event.E { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
result := make(map[uint64]*event.E) |
||||
for serial, ev := range pb.Events { |
||||
result[serial] = ProtoToEvent(ev) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// EventsToProto converts a slice of events to a slice of proto Events.
|
||||
func EventsToProto(events event.S) []*Event { |
||||
result := make([]*Event, 0, len(events)) |
||||
for _, ev := range events { |
||||
result = append(result, EventToProto(ev)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// ProtoToEvents converts a slice of proto Events to a slice of events.
|
||||
func ProtoToEvents(pbs []*Event) event.S { |
||||
result := make(event.S, 0, len(pbs)) |
||||
for _, pb := range pbs { |
||||
result = append(result, ProtoToEvent(pb)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// timeFromUnix converts a unix timestamp to time.Time (unexported).
|
||||
func timeFromUnix(unix int64) time.Time { |
||||
if unix == 0 { |
||||
return time.Time{} |
||||
} |
||||
return time.Unix(unix, 0) |
||||
} |
||||
|
||||
// TimeFromUnix converts a unix timestamp to time.Time (exported).
|
||||
func TimeFromUnix(unix int64) time.Time { |
||||
return timeFromUnix(unix) |
||||
} |
||||
|
||||
// newUint40 creates a new Uint40 with the given value.
|
||||
func newUint40(value uint64) *indextypes.Uint40 { |
||||
u := &indextypes.Uint40{} |
||||
_ = u.Set(value) |
||||
return u |
||||
} |
||||
|
||||
// BytesToTag converts a slice of byte slices to a tag.T.
|
||||
func BytesToTag(ids [][]byte) *tag.T { |
||||
t := tag.NewWithCap(len(ids)) |
||||
for _, id := range ids { |
||||
t.T = append(t.T, id) |
||||
} |
||||
return t |
||||
} |
||||
|
||||
// === Blob Storage Converters ===
|
||||
|
||||
// BlobMetadataToProto converts a database.BlobMetadata to a proto BlobMetadata.
|
||||
func BlobMetadataToProto(m *database.BlobMetadata) *BlobMetadata { |
||||
if m == nil { |
||||
return nil |
||||
} |
||||
return &BlobMetadata{ |
||||
Pubkey: m.Pubkey, |
||||
MimeType: m.MimeType, |
||||
Size: m.Size, |
||||
Uploaded: m.Uploaded, |
||||
Extension: m.Extension, |
||||
} |
||||
} |
||||
|
||||
// ProtoToBlobMetadata converts a proto BlobMetadata to a database.BlobMetadata.
|
||||
func ProtoToBlobMetadata(pb *BlobMetadata) *database.BlobMetadata { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
return &database.BlobMetadata{ |
||||
Pubkey: pb.Pubkey, |
||||
MimeType: pb.MimeType, |
||||
Size: pb.Size, |
||||
Uploaded: pb.Uploaded, |
||||
Extension: pb.Extension, |
||||
} |
||||
} |
||||
|
||||
// BlobDescriptorToProto converts a database.BlobDescriptor to a proto BlobDescriptor.
|
||||
// Note: NIP94 field is not transported via gRPC as it's generated at response time.
|
||||
func BlobDescriptorToProto(d *database.BlobDescriptor) *BlobDescriptor { |
||||
if d == nil { |
||||
return nil |
||||
} |
||||
return &BlobDescriptor{ |
||||
Url: d.URL, |
||||
Sha256: d.SHA256, |
||||
Size: d.Size, |
||||
Type: d.Type, |
||||
Uploaded: d.Uploaded, |
||||
} |
||||
} |
||||
|
||||
// ProtoToBlobDescriptor converts a proto BlobDescriptor to a database.BlobDescriptor.
|
||||
// Note: NIP94 field is not populated from proto as it's generated at response time.
|
||||
func ProtoToBlobDescriptor(pb *BlobDescriptor) *database.BlobDescriptor { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
return &database.BlobDescriptor{ |
||||
URL: pb.Url, |
||||
SHA256: pb.Sha256, |
||||
Size: pb.Size, |
||||
Type: pb.Type, |
||||
Uploaded: pb.Uploaded, |
||||
} |
||||
} |
||||
|
||||
// BlobDescriptorListToProto converts a slice of BlobDescriptors to proto.
|
||||
func BlobDescriptorListToProto(descriptors []*database.BlobDescriptor) []*BlobDescriptor { |
||||
result := make([]*BlobDescriptor, 0, len(descriptors)) |
||||
for _, d := range descriptors { |
||||
result = append(result, BlobDescriptorToProto(d)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// ProtoToBlobDescriptorList converts proto BlobDescriptors to a slice.
|
||||
func ProtoToBlobDescriptorList(pbs []*BlobDescriptor) []*database.BlobDescriptor { |
||||
result := make([]*database.BlobDescriptor, 0, len(pbs)) |
||||
for _, pb := range pbs { |
||||
result = append(result, ProtoToBlobDescriptor(pb)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// UserBlobStatsToProto converts a database.UserBlobStats to a proto UserBlobStats.
|
||||
func UserBlobStatsToProto(s *database.UserBlobStats) *UserBlobStats { |
||||
if s == nil { |
||||
return nil |
||||
} |
||||
return &UserBlobStats{ |
||||
PubkeyHex: s.PubkeyHex, |
||||
BlobCount: s.BlobCount, |
||||
TotalSizeBytes: s.TotalSizeBytes, |
||||
} |
||||
} |
||||
|
||||
// ProtoToUserBlobStats converts a proto UserBlobStats to a database.UserBlobStats.
|
||||
func ProtoToUserBlobStats(pb *UserBlobStats) *database.UserBlobStats { |
||||
if pb == nil { |
||||
return nil |
||||
} |
||||
return &database.UserBlobStats{ |
||||
PubkeyHex: pb.PubkeyHex, |
||||
BlobCount: pb.BlobCount, |
||||
TotalSizeBytes: pb.TotalSizeBytes, |
||||
} |
||||
} |
||||
|
||||
// UserBlobStatsListToProto converts a slice of UserBlobStats to proto.
|
||||
func UserBlobStatsListToProto(stats []*database.UserBlobStats) []*UserBlobStats { |
||||
result := make([]*UserBlobStats, 0, len(stats)) |
||||
for _, s := range stats { |
||||
result = append(result, UserBlobStatsToProto(s)) |
||||
} |
||||
return result |
||||
} |
||||
|
||||
// ProtoToUserBlobStatsList converts proto UserBlobStats to a slice.
|
||||
func ProtoToUserBlobStatsList(pbs []*UserBlobStats) []*database.UserBlobStats { |
||||
result := make([]*database.UserBlobStats, 0, len(pbs)) |
||||
for _, pb := range pbs { |
||||
result = append(result, ProtoToUserBlobStats(pb)) |
||||
} |
||||
return result |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,61 @@
@@ -0,0 +1,61 @@
|
||||
//go:build js && wasm
|
||||
|
||||
package wasmdb |
||||
|
||||
import ( |
||||
"errors" |
||||
|
||||
"next.orly.dev/pkg/database" |
||||
) |
||||
|
||||
// Blob storage methods - WasmDB does not support blob storage
|
||||
// These are stub implementations that return errors.
|
||||
// Blob storage is not supported in the browser environment due to
|
||||
// IndexedDB size limitations and the complexity of binary data handling.
|
||||
|
||||
var errBlobNotSupported = errors.New("blob storage not supported in WasmDB backend") |
||||
|
||||
// SaveBlob stores a blob with its metadata (not supported in WasmDB)
|
||||
func (w *W) SaveBlob(sha256Hash []byte, data []byte, pubkey []byte, mimeType string, extension string) error { |
||||
return errBlobNotSupported |
||||
} |
||||
|
||||
// GetBlob retrieves blob data by SHA256 hash (not supported in WasmDB)
|
||||
func (w *W) GetBlob(sha256Hash []byte) (data []byte, metadata *database.BlobMetadata, err error) { |
||||
return nil, nil, errBlobNotSupported |
||||
} |
||||
|
||||
// HasBlob checks if a blob exists (not supported in WasmDB)
|
||||
func (w *W) HasBlob(sha256Hash []byte) (exists bool, err error) { |
||||
return false, errBlobNotSupported |
||||
} |
||||
|
||||
// DeleteBlob deletes a blob and its metadata (not supported in WasmDB)
|
||||
func (w *W) DeleteBlob(sha256Hash []byte, pubkey []byte) error { |
||||
return errBlobNotSupported |
||||
} |
||||
|
||||
// ListBlobs lists all blobs for a given pubkey (not supported in WasmDB)
|
||||
func (w *W) ListBlobs(pubkey []byte, since, until int64) ([]*database.BlobDescriptor, error) { |
||||
return nil, errBlobNotSupported |
||||
} |
||||
|
||||
// GetBlobMetadata retrieves only metadata for a blob (not supported in WasmDB)
|
||||
func (w *W) GetBlobMetadata(sha256Hash []byte) (*database.BlobMetadata, error) { |
||||
return nil, errBlobNotSupported |
||||
} |
||||
|
||||
// GetTotalBlobStorageUsed calculates total storage used by a pubkey in MB (not supported in WasmDB)
|
||||
func (w *W) GetTotalBlobStorageUsed(pubkey []byte) (totalMB int64, err error) { |
||||
return 0, errBlobNotSupported |
||||
} |
||||
|
||||
// SaveBlobReport stores a report for a blob (not supported in WasmDB)
|
||||
func (w *W) SaveBlobReport(sha256Hash []byte, reportData []byte) error { |
||||
return errBlobNotSupported |
||||
} |
||||
|
||||
// ListAllBlobUserStats returns storage statistics for all users (not supported in WasmDB)
|
||||
func (w *W) ListAllBlobUserStats() ([]*database.UserBlobStats, error) { |
||||
return nil, errBlobNotSupported |
||||
} |
||||
@ -0,0 +1,10 @@
@@ -0,0 +1,10 @@
|
||||
version: v1 |
||||
plugins: |
||||
- plugin: go |
||||
out: ../pkg/proto |
||||
opt: |
||||
- paths=source_relative |
||||
- plugin: go-grpc |
||||
out: ../pkg/proto |
||||
opt: |
||||
- paths=source_relative |
||||
@ -0,0 +1,10 @@
@@ -0,0 +1,10 @@
|
||||
version: v1 |
||||
deps: [] |
||||
lint: |
||||
use: |
||||
- DEFAULT |
||||
except: |
||||
- PACKAGE_VERSION_SUFFIX |
||||
breaking: |
||||
use: |
||||
- FILE |
||||
@ -0,0 +1,668 @@
@@ -0,0 +1,668 @@
|
||||
syntax = "proto3"; |
||||
package orlydb.v1; |
||||
option go_package = "next.orly.dev/pkg/proto/orlydb/v1;orlydbv1"; |
||||
|
||||
import "orlydb/v1/types.proto"; |
||||
|
||||
// DatabaseService provides gRPC access to the ORLY database |
||||
service DatabaseService { |
||||
// === Lifecycle Methods === |
||||
|
||||
// GetPath returns the database file path |
||||
rpc GetPath(Empty) returns (PathResponse); |
||||
|
||||
// Sync flushes buffers to disk |
||||
rpc Sync(Empty) returns (Empty); |
||||
|
||||
// Ready returns whether the database is ready to serve requests |
||||
rpc Ready(Empty) returns (ReadyResponse); |
||||
|
||||
// SetLogLevel sets the database log level |
||||
rpc SetLogLevel(SetLogLevelRequest) returns (Empty); |
||||
|
||||
// === Event Storage === |
||||
|
||||
// SaveEvent stores an event, returns whether it already existed |
||||
rpc SaveEvent(SaveEventRequest) returns (SaveEventResponse); |
||||
|
||||
// GetSerialsFromFilter gets serial numbers matching a filter |
||||
rpc GetSerialsFromFilter(GetSerialsFromFilterRequest) returns (SerialList); |
||||
|
||||
// WouldReplaceEvent checks if an event would replace existing ones |
||||
rpc WouldReplaceEvent(WouldReplaceEventRequest) returns (WouldReplaceEventResponse); |
||||
|
||||
// === Event Queries (Streaming for large results) === |
||||
|
||||
// QueryEvents queries events matching a filter (with deletion filtering) |
||||
rpc QueryEvents(QueryEventsRequest) returns (stream EventBatch); |
||||
|
||||
// QueryAllVersions queries all versions of replaceable events |
||||
rpc QueryAllVersions(QueryEventsRequest) returns (stream EventBatch); |
||||
|
||||
// QueryEventsWithOptions provides full control over query behavior |
||||
rpc QueryEventsWithOptions(QueryEventsWithOptionsRequest) returns (stream EventBatch); |
||||
|
||||
// QueryDeleteEventsByTargetId queries deletion events for a target |
||||
rpc QueryDeleteEventsByTargetId(QueryDeleteEventsByTargetIdRequest) returns (stream EventBatch); |
||||
|
||||
// QueryForSerials queries just serial numbers matching a filter |
||||
rpc QueryForSerials(QueryEventsRequest) returns (SerialList); |
||||
|
||||
// QueryForIds queries ID/Pubkey/Timestamp triplets |
||||
rpc QueryForIds(QueryEventsRequest) returns (IdPkTsList); |
||||
|
||||
// CountEvents counts events matching a filter |
||||
rpc CountEvents(QueryEventsRequest) returns (CountEventsResponse); |
||||
|
||||
// === Event Retrieval by Serial === |
||||
|
||||
// FetchEventBySerial fetches a single event by serial |
||||
rpc FetchEventBySerial(FetchEventBySerialRequest) returns (FetchEventBySerialResponse); |
||||
|
||||
// FetchEventsBySerials fetches multiple events by serial (batch) |
||||
rpc FetchEventsBySerials(FetchEventsBySerialRequest) returns (EventMap); |
||||
|
||||
// GetSerialById gets the serial for a single event ID |
||||
rpc GetSerialById(GetSerialByIdRequest) returns (GetSerialByIdResponse); |
||||
|
||||
// GetSerialsByIds gets serials for multiple event IDs |
||||
rpc GetSerialsByIds(GetSerialsByIdsRequest) returns (SerialMap); |
||||
|
||||
// GetSerialsByRange gets serials within an index range |
||||
rpc GetSerialsByRange(GetSerialsByRangeRequest) returns (SerialList); |
||||
|
||||
// GetFullIdPubkeyBySerial gets full ID/Pubkey/Timestamp for a serial |
||||
rpc GetFullIdPubkeyBySerial(GetFullIdPubkeyBySerialRequest) returns (IdPkTs); |
||||
|
||||
// GetFullIdPubkeyBySerials gets full data for multiple serials |
||||
rpc GetFullIdPubkeyBySerials(GetFullIdPubkeyBySerialsRequest) returns (IdPkTsList); |
||||
|
||||
// === Event Deletion === |
||||
|
||||
// DeleteEvent deletes an event by ID |
||||
rpc DeleteEvent(DeleteEventRequest) returns (Empty); |
||||
|
||||
// DeleteEventBySerial deletes an event by serial |
||||
rpc DeleteEventBySerial(DeleteEventBySerialRequest) returns (Empty); |
||||
|
||||
// DeleteExpired deletes events with expired expiration tags |
||||
rpc DeleteExpired(Empty) returns (Empty); |
||||
|
||||
// ProcessDelete processes a deletion event (NIP-09) |
||||
rpc ProcessDelete(ProcessDeleteRequest) returns (Empty); |
||||
|
||||
// CheckForDeleted checks if an event was deleted |
||||
rpc CheckForDeleted(CheckForDeletedRequest) returns (Empty); |
||||
|
||||
// === Import/Export (Streaming) === |
||||
|
||||
// Import imports events from a stream |
||||
rpc Import(stream ImportChunk) returns (ImportResponse); |
||||
|
||||
// Export exports events to a stream |
||||
rpc Export(ExportRequest) returns (stream ExportChunk); |
||||
|
||||
// ImportEventsFromStrings imports events from JSON strings |
||||
rpc ImportEventsFromStrings(ImportEventsFromStringsRequest) returns (ImportResponse); |
||||
|
||||
// === Relay Identity === |
||||
|
||||
// GetRelayIdentitySecret gets the relay's secret key |
||||
rpc GetRelayIdentitySecret(Empty) returns (GetRelayIdentitySecretResponse); |
||||
|
||||
// SetRelayIdentitySecret sets the relay's secret key |
||||
rpc SetRelayIdentitySecret(SetRelayIdentitySecretRequest) returns (Empty); |
||||
|
||||
// GetOrCreateRelayIdentitySecret gets or creates the relay's secret key |
||||
rpc GetOrCreateRelayIdentitySecret(Empty) returns (GetRelayIdentitySecretResponse); |
||||
|
||||
// === Markers (Key-Value Storage) === |
||||
|
||||
// SetMarker sets a metadata marker |
||||
rpc SetMarker(SetMarkerRequest) returns (Empty); |
||||
|
||||
// GetMarker gets a metadata marker |
||||
rpc GetMarker(GetMarkerRequest) returns (GetMarkerResponse); |
||||
|
||||
// HasMarker checks if a marker exists |
||||
rpc HasMarker(HasMarkerRequest) returns (HasMarkerResponse); |
||||
|
||||
// DeleteMarker deletes a marker |
||||
rpc DeleteMarker(DeleteMarkerRequest) returns (Empty); |
||||
|
||||
// === Subscriptions (Payment-Based Access) === |
||||
|
||||
// GetSubscription gets subscription info for a pubkey |
||||
rpc GetSubscription(GetSubscriptionRequest) returns (Subscription); |
||||
|
||||
// IsSubscriptionActive checks if a subscription is active |
||||
rpc IsSubscriptionActive(IsSubscriptionActiveRequest) returns (IsSubscriptionActiveResponse); |
||||
|
||||
// ExtendSubscription extends a subscription by days |
||||
rpc ExtendSubscription(ExtendSubscriptionRequest) returns (Empty); |
||||
|
||||
// RecordPayment records a payment |
||||
rpc RecordPayment(RecordPaymentRequest) returns (Empty); |
||||
|
||||
// GetPaymentHistory gets payment history for a pubkey |
||||
rpc GetPaymentHistory(GetPaymentHistoryRequest) returns (PaymentList); |
||||
|
||||
// ExtendBlossomSubscription extends blossom storage subscription |
||||
rpc ExtendBlossomSubscription(ExtendBlossomSubscriptionRequest) returns (Empty); |
||||
|
||||
// GetBlossomStorageQuota gets blossom storage quota |
||||
rpc GetBlossomStorageQuota(GetBlossomStorageQuotaRequest) returns (GetBlossomStorageQuotaResponse); |
||||
|
||||
// IsFirstTimeUser checks if this is a first-time user |
||||
rpc IsFirstTimeUser(IsFirstTimeUserRequest) returns (IsFirstTimeUserResponse); |
||||
|
||||
// === NIP-43 Invite-Based ACL === |
||||
|
||||
// AddNIP43Member adds a member via invite code |
||||
rpc AddNIP43Member(AddNIP43MemberRequest) returns (Empty); |
||||
|
||||
// RemoveNIP43Member removes a member |
||||
rpc RemoveNIP43Member(RemoveNIP43MemberRequest) returns (Empty); |
||||
|
||||
// IsNIP43Member checks if a pubkey is a member |
||||
rpc IsNIP43Member(IsNIP43MemberRequest) returns (IsNIP43MemberResponse); |
||||
|
||||
// GetNIP43Membership gets membership details |
||||
rpc GetNIP43Membership(GetNIP43MembershipRequest) returns (NIP43Membership); |
||||
|
||||
// GetAllNIP43Members gets all members |
||||
rpc GetAllNIP43Members(Empty) returns (PubkeyList); |
||||
|
||||
// StoreInviteCode stores an invite code with expiration |
||||
rpc StoreInviteCode(StoreInviteCodeRequest) returns (Empty); |
||||
|
||||
// ValidateInviteCode validates an invite code |
||||
rpc ValidateInviteCode(ValidateInviteCodeRequest) returns (ValidateInviteCodeResponse); |
||||
|
||||
// DeleteInviteCode deletes an invite code |
||||
rpc DeleteInviteCode(DeleteInviteCodeRequest) returns (Empty); |
||||
|
||||
// PublishNIP43MembershipEvent publishes a membership event |
||||
rpc PublishNIP43MembershipEvent(PublishNIP43MembershipEventRequest) returns (Empty); |
||||
|
||||
// === Query Cache === |
||||
|
||||
// GetCachedJSON gets cached JSON for a filter |
||||
rpc GetCachedJSON(GetCachedJSONRequest) returns (GetCachedJSONResponse); |
||||
|
||||
// CacheMarshaledJSON caches JSON for a filter |
||||
rpc CacheMarshaledJSON(CacheMarshaledJSONRequest) returns (Empty); |
||||
|
||||
// GetCachedEvents gets cached events for a filter |
||||
rpc GetCachedEvents(GetCachedEventsRequest) returns (GetCachedEventsResponse); |
||||
|
||||
// CacheEvents caches events for a filter |
||||
rpc CacheEvents(CacheEventsRequest) returns (Empty); |
||||
|
||||
// InvalidateQueryCache invalidates the entire query cache |
||||
rpc InvalidateQueryCache(Empty) returns (Empty); |
||||
|
||||
// === Access Tracking === |
||||
|
||||
// RecordEventAccess records an access to an event |
||||
rpc RecordEventAccess(RecordEventAccessRequest) returns (Empty); |
||||
|
||||
// GetEventAccessInfo gets access info for an event |
||||
rpc GetEventAccessInfo(GetEventAccessInfoRequest) returns (GetEventAccessInfoResponse); |
||||
|
||||
// GetLeastAccessedEvents gets least accessed events for GC |
||||
rpc GetLeastAccessedEvents(GetLeastAccessedEventsRequest) returns (SerialList); |
||||
|
||||
// === Blob Storage (Blossom) === |
||||
|
||||
// SaveBlob stores a blob with its metadata |
||||
rpc SaveBlob(SaveBlobRequest) returns (Empty); |
||||
|
||||
// GetBlob retrieves blob data and metadata |
||||
rpc GetBlob(GetBlobRequest) returns (GetBlobResponse); |
||||
|
||||
// HasBlob checks if a blob exists |
||||
rpc HasBlob(HasBlobRequest) returns (HasBlobResponse); |
||||
|
||||
// DeleteBlob deletes a blob |
||||
rpc DeleteBlob(DeleteBlobRequest) returns (Empty); |
||||
|
||||
// ListBlobs lists blobs for a pubkey |
||||
rpc ListBlobs(ListBlobsRequest) returns (ListBlobsResponse); |
||||
|
||||
// GetBlobMetadata gets only metadata for a blob |
||||
rpc GetBlobMetadata(GetBlobMetadataRequest) returns (BlobMetadata); |
||||
|
||||
// GetTotalBlobStorageUsed gets total storage used by a pubkey in MB |
||||
rpc GetTotalBlobStorageUsed(GetTotalBlobStorageUsedRequest) returns (GetTotalBlobStorageUsedResponse); |
||||
|
||||
// SaveBlobReport stores a report for a blob (BUD-09) |
||||
rpc SaveBlobReport(SaveBlobReportRequest) returns (Empty); |
||||
|
||||
// ListAllBlobUserStats gets storage statistics for all users |
||||
rpc ListAllBlobUserStats(Empty) returns (ListAllBlobUserStatsResponse); |
||||
|
||||
// === Utility === |
||||
|
||||
// EventIdsBySerial gets event IDs by serial range |
||||
rpc EventIdsBySerial(EventIdsBySerialRequest) returns (EventIdsBySerialResponse); |
||||
|
||||
// RunMigrations runs database migrations |
||||
rpc RunMigrations(Empty) returns (Empty); |
||||
} |
||||
|
||||
// === Request/Response Messages === |
||||
|
||||
// Lifecycle |
||||
message PathResponse { |
||||
string path = 1; |
||||
} |
||||
|
||||
message ReadyResponse { |
||||
bool ready = 1; |
||||
} |
||||
|
||||
message SetLogLevelRequest { |
||||
string level = 1; |
||||
} |
||||
|
||||
// Event Storage |
||||
message SaveEventRequest { |
||||
Event event = 1; |
||||
} |
||||
|
||||
message SaveEventResponse { |
||||
bool exists = 1; |
||||
} |
||||
|
||||
message GetSerialsFromFilterRequest { |
||||
Filter filter = 1; |
||||
} |
||||
|
||||
message WouldReplaceEventRequest { |
||||
Event event = 1; |
||||
} |
||||
|
||||
message WouldReplaceEventResponse { |
||||
bool would_replace = 1; |
||||
repeated uint64 replaced_serials = 2; |
||||
} |
||||
|
||||
// Event Queries |
||||
message QueryEventsRequest { |
||||
Filter filter = 1; |
||||
} |
||||
|
||||
message QueryEventsWithOptionsRequest { |
||||
Filter filter = 1; |
||||
bool include_delete_events = 2; |
||||
bool show_all_versions = 3; |
||||
} |
||||
|
||||
message QueryDeleteEventsByTargetIdRequest { |
||||
bytes target_event_id = 1; |
||||
} |
||||
|
||||
message CountEventsResponse { |
||||
int32 count = 1; |
||||
bool approximate = 2; |
||||
} |
||||
|
||||
// Event Retrieval |
||||
message FetchEventBySerialRequest { |
||||
uint64 serial = 1; |
||||
} |
||||
|
||||
message FetchEventBySerialResponse { |
||||
Event event = 1; |
||||
bool found = 2; |
||||
} |
||||
|
||||
message FetchEventsBySerialRequest { |
||||
repeated uint64 serials = 1; |
||||
} |
||||
|
||||
message GetSerialByIdRequest { |
||||
bytes id = 1; |
||||
} |
||||
|
||||
message GetSerialByIdResponse { |
||||
uint64 serial = 1; |
||||
bool found = 2; |
||||
} |
||||
|
||||
message GetSerialsByIdsRequest { |
||||
repeated bytes ids = 1; |
||||
} |
||||
|
||||
message GetSerialsByRangeRequest { |
||||
Range range = 1; |
||||
} |
||||
|
||||
message GetFullIdPubkeyBySerialRequest { |
||||
uint64 serial = 1; |
||||
} |
||||
|
||||
message GetFullIdPubkeyBySerialsRequest { |
||||
repeated uint64 serials = 1; |
||||
} |
||||
|
||||
// Event Deletion |
||||
message DeleteEventRequest { |
||||
bytes event_id = 1; |
||||
} |
||||
|
||||
message DeleteEventBySerialRequest { |
||||
uint64 serial = 1; |
||||
Event event = 2; |
||||
} |
||||
|
||||
message ProcessDeleteRequest { |
||||
Event event = 1; |
||||
repeated bytes admins = 2; |
||||
} |
||||
|
||||
message CheckForDeletedRequest { |
||||
Event event = 1; |
||||
repeated bytes admins = 2; |
||||
} |
||||
|
||||
// Import/Export |
||||
message ImportChunk { |
||||
bytes data = 1; |
||||
} |
||||
|
||||
message ImportResponse { |
||||
int64 events_imported = 1; |
||||
int64 events_skipped = 2; |
||||
} |
||||
|
||||
message ExportRequest { |
||||
repeated bytes pubkeys = 1; |
||||
} |
||||
|
||||
message ExportChunk { |
||||
bytes data = 1; |
||||
} |
||||
|
||||
message ImportEventsFromStringsRequest { |
||||
repeated string event_jsons = 1; |
||||
bool check_policy = 2; |
||||
} |
||||
|
||||
// Relay Identity |
||||
message GetRelayIdentitySecretResponse { |
||||
bytes secret_key = 1; |
||||
} |
||||
|
||||
message SetRelayIdentitySecretRequest { |
||||
bytes secret_key = 1; |
||||
} |
||||
|
||||
// Markers |
||||
message SetMarkerRequest { |
||||
string key = 1; |
||||
bytes value = 2; |
||||
} |
||||
|
||||
message GetMarkerRequest { |
||||
string key = 1; |
||||
} |
||||
|
||||
message GetMarkerResponse { |
||||
bytes value = 1; |
||||
bool found = 2; |
||||
} |
||||
|
||||
message HasMarkerRequest { |
||||
string key = 1; |
||||
} |
||||
|
||||
message HasMarkerResponse { |
||||
bool exists = 1; |
||||
} |
||||
|
||||
message DeleteMarkerRequest { |
||||
string key = 1; |
||||
} |
||||
|
||||
// Subscriptions |
||||
message GetSubscriptionRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message IsSubscriptionActiveRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message IsSubscriptionActiveResponse { |
||||
bool active = 1; |
||||
} |
||||
|
||||
message ExtendSubscriptionRequest { |
||||
bytes pubkey = 1; |
||||
int32 days = 2; |
||||
} |
||||
|
||||
message RecordPaymentRequest { |
||||
bytes pubkey = 1; |
||||
int64 amount = 2; |
||||
string invoice = 3; |
||||
string preimage = 4; |
||||
} |
||||
|
||||
message GetPaymentHistoryRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message ExtendBlossomSubscriptionRequest { |
||||
bytes pubkey = 1; |
||||
string tier = 2; |
||||
int64 storage_mb = 3; |
||||
int32 days_extended = 4; |
||||
} |
||||
|
||||
message GetBlossomStorageQuotaRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message GetBlossomStorageQuotaResponse { |
||||
int64 quota_mb = 1; |
||||
} |
||||
|
||||
message IsFirstTimeUserRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message IsFirstTimeUserResponse { |
||||
bool first_time = 1; |
||||
} |
||||
|
||||
// NIP-43 |
||||
message AddNIP43MemberRequest { |
||||
bytes pubkey = 1; |
||||
string invite_code = 2; |
||||
} |
||||
|
||||
message RemoveNIP43MemberRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message IsNIP43MemberRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message IsNIP43MemberResponse { |
||||
bool is_member = 1; |
||||
} |
||||
|
||||
message GetNIP43MembershipRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message StoreInviteCodeRequest { |
||||
string code = 1; |
||||
int64 expires_at = 2; |
||||
} |
||||
|
||||
message ValidateInviteCodeRequest { |
||||
string code = 1; |
||||
} |
||||
|
||||
message ValidateInviteCodeResponse { |
||||
bool valid = 1; |
||||
} |
||||
|
||||
message DeleteInviteCodeRequest { |
||||
string code = 1; |
||||
} |
||||
|
||||
message PublishNIP43MembershipEventRequest { |
||||
int32 kind = 1; |
||||
bytes pubkey = 2; |
||||
} |
||||
|
||||
// Query Cache |
||||
message GetCachedJSONRequest { |
||||
Filter filter = 1; |
||||
} |
||||
|
||||
message GetCachedJSONResponse { |
||||
repeated bytes json_items = 1; |
||||
bool found = 2; |
||||
} |
||||
|
||||
message CacheMarshaledJSONRequest { |
||||
Filter filter = 1; |
||||
repeated bytes json_items = 2; |
||||
} |
||||
|
||||
message GetCachedEventsRequest { |
||||
Filter filter = 1; |
||||
} |
||||
|
||||
message GetCachedEventsResponse { |
||||
repeated Event events = 1; |
||||
bool found = 2; |
||||
} |
||||
|
||||
message CacheEventsRequest { |
||||
Filter filter = 1; |
||||
repeated Event events = 2; |
||||
} |
||||
|
||||
// Access Tracking |
||||
message RecordEventAccessRequest { |
||||
uint64 serial = 1; |
||||
string connection_id = 2; |
||||
} |
||||
|
||||
message GetEventAccessInfoRequest { |
||||
uint64 serial = 1; |
||||
} |
||||
|
||||
message GetEventAccessInfoResponse { |
||||
int64 last_access = 1; |
||||
uint32 access_count = 2; |
||||
} |
||||
|
||||
message GetLeastAccessedEventsRequest { |
||||
int32 limit = 1; |
||||
int64 min_age_sec = 2; |
||||
} |
||||
|
||||
// Utility |
||||
message EventIdsBySerialRequest { |
||||
uint64 start = 1; |
||||
int32 count = 2; |
||||
} |
||||
|
||||
message EventIdsBySerialResponse { |
||||
repeated uint64 event_ids = 1; |
||||
} |
||||
|
||||
// === Blob Storage Messages === |
||||
|
||||
message BlobMetadata { |
||||
bytes pubkey = 1; |
||||
string mime_type = 2; |
||||
int64 size = 3; |
||||
int64 uploaded = 4; // Unix timestamp |
||||
string extension = 5; |
||||
} |
||||
|
||||
message BlobDescriptor { |
||||
string url = 1; |
||||
string sha256 = 2; |
||||
int64 size = 3; |
||||
string type = 4; // MIME type |
||||
int64 uploaded = 5; |
||||
} |
||||
|
||||
message SaveBlobRequest { |
||||
bytes sha256_hash = 1; |
||||
bytes data = 2; |
||||
bytes pubkey = 3; |
||||
string mime_type = 4; |
||||
string extension = 5; |
||||
} |
||||
|
||||
message GetBlobRequest { |
||||
bytes sha256_hash = 1; |
||||
} |
||||
|
||||
message GetBlobResponse { |
||||
bool found = 1; |
||||
bytes data = 2; |
||||
BlobMetadata metadata = 3; |
||||
} |
||||
|
||||
message HasBlobRequest { |
||||
bytes sha256_hash = 1; |
||||
} |
||||
|
||||
message HasBlobResponse { |
||||
bool exists = 1; |
||||
} |
||||
|
||||
message DeleteBlobRequest { |
||||
bytes sha256_hash = 1; |
||||
bytes pubkey = 2; |
||||
} |
||||
|
||||
message ListBlobsRequest { |
||||
bytes pubkey = 1; |
||||
int64 since = 2; |
||||
int64 until = 3; |
||||
} |
||||
|
||||
message ListBlobsResponse { |
||||
repeated BlobDescriptor descriptors = 1; |
||||
} |
||||
|
||||
message GetBlobMetadataRequest { |
||||
bytes sha256_hash = 1; |
||||
} |
||||
|
||||
message GetTotalBlobStorageUsedRequest { |
||||
bytes pubkey = 1; |
||||
} |
||||
|
||||
message GetTotalBlobStorageUsedResponse { |
||||
int64 total_mb = 1; |
||||
} |
||||
|
||||
message SaveBlobReportRequest { |
||||
bytes sha256_hash = 1; |
||||
bytes report_data = 2; |
||||
} |
||||
|
||||
message UserBlobStats { |
||||
string pubkey_hex = 1; |
||||
int64 blob_count = 2; |
||||
int64 total_size_bytes = 3; |
||||
} |
||||
|
||||
message ListAllBlobUserStatsResponse { |
||||
repeated UserBlobStats stats = 1; |
||||
} |
||||
@ -0,0 +1,121 @@
@@ -0,0 +1,121 @@
|
||||
syntax = "proto3"; |
||||
package orlydb.v1; |
||||
option go_package = "next.orly.dev/pkg/proto/orlydb/v1;orlydbv1"; |
||||
|
||||
// Empty is used for requests/responses with no data |
||||
message Empty {} |
||||
|
||||
// Event represents a Nostr event |
||||
// Binary fields (id, pubkey, sig) are stored as raw bytes for efficiency |
||||
message Event { |
||||
bytes id = 1; // 32 bytes SHA256 hash |
||||
bytes pubkey = 2; // 32 bytes public key |
||||
int64 created_at = 3; // UNIX timestamp |
||||
uint32 kind = 4; // Event kind (uint16 in Go, promoted to uint32) |
||||
repeated Tag tags = 5; // Event tags |
||||
bytes content = 6; // Arbitrary content (may be binary) |
||||
bytes sig = 7; // 64 bytes Schnorr signature |
||||
} |
||||
|
||||
// Tag represents a Nostr tag (array of byte slices) |
||||
// Values are bytes to preserve binary data in e/p tags (33-byte format) |
||||
message Tag { |
||||
repeated bytes values = 1; |
||||
} |
||||
|
||||
// TagSet represents a set of tag values for filtering |
||||
message TagSet { |
||||
repeated bytes values = 1; |
||||
} |
||||
|
||||
// Filter represents a Nostr query filter (NIP-01) |
||||
message Filter { |
||||
repeated bytes ids = 1; // Event IDs to match (32 bytes each) |
||||
repeated uint32 kinds = 2; // Kinds to match |
||||
repeated bytes authors = 3; // Author pubkeys (32 bytes each) |
||||
map<string, TagSet> tags = 4; // Tag filters (#e, #p, #t, etc.) |
||||
optional int64 since = 5; // Created after timestamp |
||||
optional int64 until = 6; // Created before timestamp |
||||
optional bytes search = 7; // Full-text search query (NIP-50) |
||||
optional uint32 limit = 8; // Max results |
||||
} |
||||
|
||||
// Uint40 represents a 40-bit serial number |
||||
// Packed into uint64 with upper 24 bits unused |
||||
message Uint40 { |
||||
uint64 value = 1; |
||||
} |
||||
|
||||
// IdPkTs holds event reference data (ID, Pubkey, Timestamp, Serial) |
||||
message IdPkTs { |
||||
bytes id = 1; // 32 bytes event ID |
||||
bytes pubkey = 2; // 32 bytes author pubkey |
||||
int64 timestamp = 3; // Created timestamp |
||||
uint64 serial = 4; // Database serial number |
||||
} |
||||
|
||||
// Range represents an index range for serial queries |
||||
message Range { |
||||
bytes prefix = 1; // Index prefix |
||||
uint64 start = 2; // Start serial |
||||
uint64 end = 3; // End serial |
||||
} |
||||
|
||||
// Subscription represents payment-based access control data |
||||
message Subscription { |
||||
bytes pubkey = 1; |
||||
int64 trial_end = 2; |
||||
int64 paid_until = 3; |
||||
string blossom_level = 4; |
||||
int64 blossom_storage_mb = 5; |
||||
} |
||||
|
||||
// Payment represents a payment record |
||||
message Payment { |
||||
int64 amount = 1; |
||||
int64 timestamp = 2; |
||||
string invoice = 3; |
||||
string preimage = 4; |
||||
} |
||||
|
||||
// NIP43Membership represents NIP-43 invite-based ACL membership |
||||
message NIP43Membership { |
||||
bytes pubkey = 1; |
||||
int64 added_at = 2; |
||||
string invite_code = 3; |
||||
} |
||||
|
||||
// EventBatch is used for streaming query results |
||||
message EventBatch { |
||||
repeated Event events = 1; |
||||
} |
||||
|
||||
// SerialList is used for returning lists of serials |
||||
message SerialList { |
||||
repeated uint64 serials = 1; |
||||
} |
||||
|
||||
// IdPkTsList is used for returning lists of IdPkTs |
||||
message IdPkTsList { |
||||
repeated IdPkTs items = 1; |
||||
} |
||||
|
||||
// EventMap maps serial numbers to events |
||||
message EventMap { |
||||
map<uint64, Event> events = 1; |
||||
} |
||||
|
||||
// SerialMap maps string IDs to serial numbers |
||||
message SerialMap { |
||||
map<string, uint64> serials = 1; |
||||
} |
||||
|
||||
// PaymentList is used for returning payment history |
||||
message PaymentList { |
||||
repeated Payment payments = 1; |
||||
} |
||||
|
||||
// PubkeyList is used for returning lists of pubkeys |
||||
message PubkeyList { |
||||
repeated bytes pubkeys = 1; |
||||
} |
||||
Loading…
Reference in new issue