From 71f5a91b76fca22a5c7e4256c3f4bb622c30dc1d Mon Sep 17 00:00:00 2001 From: Silberengel Date: Sat, 7 Feb 2026 16:52:14 +0100 Subject: [PATCH] update docs and scripts fix memory leaks --- Dockerfile.with-web | 4 +- README.md | 182 +++++++++++++++++++++++++++++++-- app/handle-negentropy.go | 9 ++ app/handle-req.go | 40 ++++++-- app/server.go | 96 ++++++++++++----- build-image-v0.58.5.sh | 16 +++ cmd/aggregator/main.go | 12 ++- package-lock.json | 6 ++ pkg/spider/directory.go | 24 +++-- pkg/spider/spider.go | 36 +++++-- run-orly-docker.sh | 18 ++-- scripts/update-embedded-web.sh | 17 +-- 12 files changed, 381 insertions(+), 79 deletions(-) create mode 100644 package-lock.json diff --git a/Dockerfile.with-web b/Dockerfile.with-web index d8ff176..b6f3664 100644 --- a/Dockerfile.with-web +++ b/Dockerfile.with-web @@ -74,7 +74,9 @@ HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ ENV ORLY_LISTEN=0.0.0.0 \ ORLY_PORT=3334 \ ORLY_DATA_DIR=/data \ - ORLY_LOG_LEVEL=info + ORLY_LOG_LEVEL=info \ + ORLY_WEB_DISABLE=false \ + ORLY_WEB_DEV_PROXY_URL="" # Run the binary ENTRYPOINT ["/app/orly"] diff --git a/README.md b/README.md index c6bf071..3760048 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ See [docs/IPC_SYNC_SERVICES.md](./docs/IPC_SYNC_SERVICES.md) for detailed API do - [Building with Web UI](#building-with-web-ui) - [Core Features](#core-features) - [Web UI](#web-ui) + - [Event Import](#event-import) + - [Event Streaming and Synchronization](#event-streaming-and-synchronization) - [Sprocket Event Processing](#sprocket-event-processing) - [Policy System](#policy-system) - [Deployment](#deployment) @@ -122,7 +124,7 @@ ORLY is a standard Go application that can be built using the Go toolchain. - Go 1.25.3 or later - Git -- For web UI: [Bun](https://bun.sh/) JavaScript runtime +- For web UI: Node.js and npm (JavaScript runtime and package manager) ### Basic Build @@ -141,8 +143,8 @@ To build with the embedded web interface: ```bash # Build the Svelte web application cd app/web -bun install -bun run build +npm install +npm run build # Build the Go binary from project root cd ../../ @@ -156,7 +158,7 @@ The recommended way to build and embed the web UI is using the provided script: ``` This script will: -- Build the Svelte app in `app/web` to `app/web/dist` using Bun (preferred) or fall back to npm/yarn/pnpm +- Build the Svelte app in `app/web` to `app/web/dist` using npm (or fall back to yarn/pnpm if npm is not available) - Run `go install` from the repository root so the binary picks up the new embedded assets - Automatically detect and use the best available JavaScript package manager @@ -167,8 +169,8 @@ For manual builds, you can also use: # build.sh echo "Building Svelte app..." cd app/web -bun install -bun run build +npm install +npm run build echo "Building Go binary..." cd ../../ @@ -210,8 +212,8 @@ For development with hot-reloading, ORLY can proxy web requests to a local dev s ```bash cd app/web -bun install -bun run dev +npm install +npm run dev ``` Note the port sirv is listening on (e.g., `http://localhost:8080`). @@ -244,7 +246,7 @@ Example with the relay on port 3334 and sirv on port 8080: ```bash # Terminal 1: Dev server -cd app/web && bun run dev +cd app/web && npm run dev # Output: Your application is ready~! # Local: http://localhost:8080 @@ -259,6 +261,168 @@ export ORLY_PORT=3334 If you only want to disable the embedded web UI (without proxying to a dev server), just set `ORLY_WEB_DISABLE=true` without setting `ORLY_WEB_DEV_PROXY_URL`. The relay will return 404 for web UI requests while still handling WebSocket and API requests. +### Event Import + +ORLY provides flexible event import capabilities for bulk data migration and synchronization. The import system supports both single files and entire directories, with efficient disk-based buffering for large datasets. + +#### Features + +- **Large File Support**: Handles multi-gigabyte files efficiently with disk buffering +- **Folder Import**: Import entire directories of JSONL files in one operation +- **Localhost Authentication**: Automatic authentication for localhost requests using `$NOSTR_PRIVATE_KEY` +- **Progress Tracking**: Real-time progress logging for long-running imports +- **Error Handling**: Continues processing even if individual events fail + +#### HTTP API Import + +Import events via the `/api/import` endpoint: + +**Single File Upload:** + +```bash +# Using multipart form data +curl -X POST http://localhost:3334/api/import \ + -H "Authorization: Nostr " \ + -F "file=@events.jsonl" + +# Using raw body +curl -X POST http://localhost:3334/api/import \ + -H "Authorization: Nostr " \ + -H "Content-Type: application/x-ndjson" \ + --data-binary @events.jsonl +``` + +**Folder Import:** + +Import all `.jsonl` files from a directory: + +```bash +curl -X POST "http://localhost:3334/api/import?folder=../scripts/exports" \ + -H "Authorization: Nostr " +``` + +The folder import will: +- Process all `.jsonl` files in the specified directory +- Return a summary with results for each file +- Continue processing even if individual files fail + +**Localhost Authentication:** + +For localhost requests, you can use the `$NOSTR_PRIVATE_KEY` environment variable instead of NIP-98 authentication: + +```bash +export NOSTR_PRIVATE_KEY=nsec1your_private_key_here + +# Import from localhost (no NIP-98 header needed) +curl -X POST http://localhost:3334/api/import \ + -F "file=@events.jsonl" +``` + +**Security:** Localhost authentication only works for requests from `127.0.0.1`, `::1`, or `localhost`. Folder imports are restricted to allowed directories (e.g., `../scripts/exports`) for security. + +#### Command-Line Import + +For direct database access, use the `orly db import` command: + +```bash +# Import a single file directly to the database +orly db import events.jsonl + +# Or specify with flag +orly db import --file /path/to/large-export.jsonl +``` + +This bypasses the HTTP API and is faster for very large imports when you have direct database access. + +#### Import Format + +Events must be in JSONL (JSON Lines) format - one JSON event per line: + +```json +{"id":"abc123...","pubkey":"def456...","kind":1,"created_at":1234567890,"content":"Hello","sig":"..."} +{"id":"xyz789...","pubkey":"ghi012...","kind":1,"created_at":1234567891,"content":"World","sig":"..."} +``` + +### Event Streaming and Synchronization + +ORLY includes powerful event streaming capabilities to forward events from a local relay to remote relays, supporting both standard Nostr protocol and efficient negentropy-based synchronization. + +#### Features + +- **WebSocket Streaming**: Stream events to any Nostr-compatible relay +- **Negentropy Support**: Efficient set reconciliation using NIP-77 protocol +- **Batch Processing**: Processes events in batches for optimal performance +- **Progress Tracking**: Real-time logging of streaming progress +- **Error Recovery**: Handles network failures gracefully + +#### HTTP API Streaming + +Stream events from your local relay to a remote relay via the `/api/stream-to-relay` endpoint: + +**Basic Streaming:** + +```bash +curl -X POST "http://localhost:3334/api/stream-to-relay?target=wss://remote-relay.com" \ + -H "Authorization: Nostr " +``` + +**Default Target:** + +If no target is specified, defaults to `wss://orly-relay.imwald.eu`: + +```bash +curl -X POST http://localhost:3334/api/stream-to-relay \ + -H "Authorization: Nostr " +``` + +**Localhost Authentication:** + +For localhost requests, use `$NOSTR_PRIVATE_KEY`: + +```bash +export NOSTR_PRIVATE_KEY=nsec1your_private_key_here + +curl -X POST "http://localhost:3334/api/stream-to-relay?target=wss://remote-relay.com" +``` + +**How It Works:** + +1. The endpoint accepts the request and returns immediately with a status message +2. Streaming happens asynchronously in the background +3. All events from the local database are queried and streamed to the target relay +4. Events are sent in batches using standard Nostr EVENT messages +5. Progress is logged to the relay's log output + +**Example Workflow:** + +```bash +# 1. Import events to local relay +curl -X POST http://localhost:3334/api/import \ + -F "file=@book-events.jsonl" + +# 2. Stream events to remote relay +curl -X POST "http://localhost:3334/api/stream-to-relay?target=wss://orly-relay.imwald.eu" \ + -H "Authorization: Nostr " +``` + +#### Negentropy Synchronization + +For more efficient synchronization, you can use negentropy (NIP-77) set reconciliation: + +```bash +curl -X POST "http://localhost:3334/api/stream-to-relay?target=wss://remote-relay.com&use_negentropy=true" \ + -H "Authorization: Nostr " +``` + +**Note:** Full negentropy support requires the target relay to be configured as a peer in the sync manager. For ad-hoc streaming, use the standard EVENT streaming method. + +#### Use Cases + +- **Data Migration**: Move events from one relay to another +- **Backup Synchronization**: Keep multiple relays in sync +- **Local to Remote**: Transfer events from a local development relay to production +- **Relay Consolidation**: Merge events from multiple sources + ### Sprocket Event Processing ORLY includes a powerful sprocket system for external event processing scripts. Sprocket scripts enable custom filtering, validation, and processing logic for Nostr events before storage. diff --git a/app/handle-negentropy.go b/app/handle-negentropy.go index 3822084..8d6724f 100644 --- a/app/handle-negentropy.go +++ b/app/handle-negentropy.go @@ -314,6 +314,15 @@ func (l *Listener) sendEventsForIDs(subscriptionID string, ids [][]byte) error { return err } + // Ensure events are freed after use + defer func() { + for _, ev := range events { + if ev != nil { + ev.Free() + } + } + }() + // Send each event via EVENT envelope with subscription ID sent := 0 for _, ev := range events { diff --git a/app/handle-req.go b/app/handle-req.go index 6ecde13..d4080de 100644 --- a/app/handle-req.go +++ b/app/handle-req.go @@ -8,10 +8,6 @@ import ( "strings" "time" - "github.com/dgraph-io/badger/v4" - "lol.mleku.dev/chk" - "lol.mleku.dev/log" - "next.orly.dev/pkg/acl" "git.mleku.dev/mleku/nostr/encoders/bech32encoding" "git.mleku.dev/mleku/nostr/encoders/envelopes/authenvelope" "git.mleku.dev/mleku/nostr/encoders/envelopes/closedenvelope" @@ -24,12 +20,16 @@ import ( "git.mleku.dev/mleku/nostr/encoders/kind" "git.mleku.dev/mleku/nostr/encoders/reason" "git.mleku.dev/mleku/nostr/encoders/tag" + "git.mleku.dev/mleku/nostr/utils/normalize" + "git.mleku.dev/mleku/nostr/utils/pointers" + "github.com/dgraph-io/badger/v4" + "lol.mleku.dev/chk" + "lol.mleku.dev/log" + "next.orly.dev/pkg/acl" "next.orly.dev/pkg/policy" "next.orly.dev/pkg/protocol/graph" "next.orly.dev/pkg/protocol/nip43" "next.orly.dev/pkg/protocol/publish" - "git.mleku.dev/mleku/nostr/utils/normalize" - "git.mleku.dev/mleku/nostr/utils/pointers" ) func (l *Listener) HandleReq(msg []byte) (err error) { @@ -292,15 +292,33 @@ func (l *Listener) HandleReq(msg []byte) (err error) { for _, ev := range cachedEvents { var res *eventenvelope.Result if res, err = eventenvelope.NewResultWith(env.Subscription, ev); chk.E(err) { + // Free remaining cached events on error + for _, remainingEv := range cachedEvents { + if remainingEv != nil { + remainingEv.Free() + } + } return } if err = res.Write(l); err != nil { if !strings.Contains(err.Error(), "context canceled") { chk.E(err) } + // Free remaining cached events on error + for _, remainingEv := range cachedEvents { + if remainingEv != nil { + remainingEv.Free() + } + } return } } + // Free cached events after sending + for _, ev := range cachedEvents { + if ev != nil { + ev.Free() + } + } // Send EOSE if err = eoseenvelope.NewFrom(env.Subscription).Write(l); chk.E(err) { return @@ -877,6 +895,7 @@ func (l *Listener) HandleReq(msg []byte) (err error) { var err error if res, err = eventenvelope.NewResultWith(subID, ev); chk.E(err) { log.E.F("failed to create event envelope for subscription %s: %v", subID, err) + ev.Free() // Free event on error continue } @@ -885,13 +904,20 @@ func (l *Listener) HandleReq(msg []byte) (err error) { if !strings.Contains(err.Error(), "context canceled") { log.E.F("failed to write event to subscription %s @ %s: %v", subID, l.remote, err) } + ev.Free() // Free event on write error // Don't return here - write errors shouldn't kill the subscription // The connection cleanup will handle removing the subscription continue } + // Capture event ID before freeing for logging + eventID := hexenc.Enc(ev.ID) + + // Free event after successfully sending + ev.Free() + log.D.F("delivered real-time event %s to subscription %s @ %s", - hexenc.Enc(ev.ID), subID, l.remote) + eventID, subID, l.remote) } } }() diff --git a/app/server.go b/app/server.go index 5648804..270987b 100644 --- a/app/server.go +++ b/app/server.go @@ -16,15 +16,16 @@ import ( "sync" "time" - "github.com/gorilla/websocket" "git.mleku.dev/mleku/nostr/encoders/bech32encoding" "git.mleku.dev/mleku/nostr/encoders/event" "git.mleku.dev/mleku/nostr/encoders/filter" "git.mleku.dev/mleku/nostr/encoders/hex" "git.mleku.dev/mleku/nostr/encoders/tag" + "git.mleku.dev/mleku/nostr/encoders/timestamp" "git.mleku.dev/mleku/nostr/httpauth" "git.mleku.dev/mleku/nostr/interfaces/signer/p8k" "git.mleku.dev/mleku/nostr/protocol/auth" + "github.com/gorilla/websocket" "lol.mleku.dev/chk" "next.orly.dev/app/branding" "next.orly.dev/app/config" @@ -1116,6 +1117,12 @@ func (s *Server) handleEventsMine(w http.ResponseWriter, r *http.Request) { // Marshal and write the response jsonData, err := json.Marshal(response) if chk.E(err) { + // Free events before returning error + for _, ev := range events { + if ev != nil { + ev.Free() + } + } http.Error( w, "Error generating response", http.StatusInternalServerError, ) @@ -1123,6 +1130,13 @@ func (s *Server) handleEventsMine(w http.ResponseWriter, r *http.Request) { } w.Write(jsonData) + + // Free events after successfully sending response + for _, ev := range events { + if ev != nil { + ev.Free() + } + } } // authenticateLocalhost authenticates requests from localhost using $NOSTR_PRIVATE_KEY @@ -1241,7 +1255,7 @@ func (s *Server) handleImportFolder(w http.ResponseWriter, r *http.Request, fold "./scripts/exports", "scripts/exports", } - + isAllowed := false for _, prefix := range allowedPrefixes { if strings.HasPrefix(folderPath, prefix) { @@ -1249,7 +1263,7 @@ func (s *Server) handleImportFolder(w http.ResponseWriter, r *http.Request, fold break } } - + // Also allow absolute paths that are explicitly in allowed directories if !isAllowed && filepath.IsAbs(folderPath) { // Check if it's under a reasonable directory (e.g., user's home or current working directory) @@ -1257,7 +1271,7 @@ func (s *Server) handleImportFolder(w http.ResponseWriter, r *http.Request, fold http.Error(w, "Absolute paths not allowed for security reasons", http.StatusForbidden) return } - + if !isAllowed { http.Error(w, "Folder path not in allowed directories", http.StatusForbidden) return @@ -1425,6 +1439,7 @@ func (s *Server) handleStreamToRelay(w http.ResponseWriter, r *http.Request) { } // streamEvents streams all events from local relay to remote relay using EVENT messages +// Uses pagination to avoid loading all events into memory at once func (s *Server) streamEvents(ctx context.Context, targetURL string) error { // Connect to remote relay dialer := websocket.Dialer{ @@ -1439,47 +1454,78 @@ func (s *Server) streamEvents(ctx context.Context, targetURL string) error { log.Printf("Connected to remote relay: %s", targetURL) - // Query all events from local database - f := filter.New() // Empty filter = all events - events, err := s.DB.QueryEvents(ctx, f) - if err != nil { - return fmt.Errorf("failed to query events: %w", err) - } + // Stream events in batches to avoid memory exhaustion + // Query events in chunks using timestamp-based pagination + const batchLimit = 1000 // Query 1000 events at a time + var since *timestamp.T + var totalSent int64 + + for { + // Create filter for this batch + f := filter.New() + if since != nil { + f.Since = since + } + limit := uint(batchLimit) + f.Limit = &limit - log.Printf("Streaming %d events to %s", len(events), targetURL) + // Query batch of events + events, err := s.DB.QueryEvents(ctx, f) + if err != nil { + return fmt.Errorf("failed to query events: %w", err) + } - // Stream events in batches - batchSize := 100 - sent := 0 - for i := 0; i < len(events); i += batchSize { - end := i + batchSize - if end > len(events) { - end = len(events) + if len(events) == 0 { + // No more events + break } - for j := i; j < end; j++ { - ev := events[j] + // Stream this batch - ensure all events are freed + processedCount := 0 + for i, ev := range events { if ev == nil { continue } + // Capture timestamp before freeing + createdAt := ev.CreatedAt + // Send EVENT message: ["EVENT", event] eventMsg := []interface{}{"EVENT", ev} if err := conn.WriteJSON(eventMsg); err != nil { + // Free this event and all remaining events in the batch + ev.Free() + for j := i + 1; j < len(events); j++ { + if events[j] != nil { + events[j].Free() + } + } return fmt.Errorf("failed to send event: %w", err) } - sent++ - if sent%1000 == 0 { - log.Printf("Streamed %d/%d events to %s", sent, len(events), targetURL) + // Free the event to return it to the pool + ev.Free() + processedCount++ + + totalSent++ + if totalSent%1000 == 0 { + log.Printf("Streamed %d events to %s", totalSent, targetURL) } + + // Update since timestamp for next batch (use created_at + 1 to avoid duplicates) + since = timestamp.FromUnix(createdAt + 1) + } + + // If we got fewer events than the limit, we've reached the end + if len(events) < batchLimit { + break } // Small delay between batches to avoid overwhelming the remote relay time.Sleep(100 * time.Millisecond) } - log.Printf("Successfully streamed %d events to %s", sent, targetURL) + log.Printf("Successfully streamed %d events to %s", totalSent, targetURL) return nil } @@ -1492,7 +1538,7 @@ func (s *Server) streamWithNegentropy(ctx context.Context, targetURL string) (in // 2. Creating negentropy instance // 3. Performing NIP-77 protocol exchange // 4. Fetching and pushing events - + // This is a placeholder - the sync manager's negentropy functionality // should be used via the existing sync infrastructure return 0, fmt.Errorf("negentropy sync via HTTP endpoint not yet fully implemented. Use the sync manager's peer configuration or implement direct negentropy protocol") diff --git a/build-image-v0.58.5.sh b/build-image-v0.58.5.sh index b2bd5d2..b37140b 100755 --- a/build-image-v0.58.5.sh +++ b/build-image-v0.58.5.sh @@ -48,6 +48,22 @@ echo "" > app/web/dist/in echo "Using Dockerfile.with-web (will build web UI in Docker with latest changes)..." DOCKERFILE="Dockerfile.with-web" +# Ensure dev proxy is disabled in Dockerfile to prevent 502 errors +echo "Ensuring dev proxy is disabled in Dockerfile..." +# Remove any existing ORLY_WEB lines to avoid duplicates +sed -i '/ORLY_WEB_DISABLE/d' "$DOCKERFILE" +sed -i '/ORLY_WEB_DEV_PROXY_URL/d' "$DOCKERFILE" +# Remove trailing backslash from ORLY_LOG_LEVEL if it exists (we'll add it back) +sed -i 's/^\( ORLY_LOG_LEVEL=info\) \\$/\1/' "$DOCKERFILE" +# Now add the new lines properly formatted +awk '/^ ORLY_LOG_LEVEL=info$/ { + print $0 " \\" + print " ORLY_WEB_DISABLE=false \\" + print " ORLY_WEB_DEV_PROXY_URL=\"\"" + next + } 1' "$DOCKERFILE" > "$DOCKERFILE.tmp" && mv "$DOCKERFILE.tmp" "$DOCKERFILE" +echo "Added ORLY_WEB_DISABLE=false and ORLY_WEB_DEV_PROXY_URL=\"\" to Dockerfile" + # Check if local nostr clone exists and prepare it for Docker build NOSTR_PATH="${NOSTR_PATH:-/home/firefly/Dokumente/repos/nostr}" mkdir -p .docker-build-context diff --git a/cmd/aggregator/main.go b/cmd/aggregator/main.go index 42cf752..cf71d5b 100644 --- a/cmd/aggregator/main.go +++ b/cmd/aggregator/main.go @@ -15,10 +15,6 @@ import ( "sync" "time" - "lol.mleku.dev/chk" - "lol.mleku.dev/log" - "git.mleku.dev/mleku/nostr/interfaces/signer/p8k" - "github.com/minio/sha256-simd" "git.mleku.dev/mleku/nostr/encoders/bech32encoding" "git.mleku.dev/mleku/nostr/encoders/event" "git.mleku.dev/mleku/nostr/encoders/filter" @@ -27,7 +23,11 @@ import ( "git.mleku.dev/mleku/nostr/encoders/tag" "git.mleku.dev/mleku/nostr/encoders/timestamp" "git.mleku.dev/mleku/nostr/interfaces/signer" + "git.mleku.dev/mleku/nostr/interfaces/signer/p8k" "git.mleku.dev/mleku/nostr/ws" + "github.com/minio/sha256-simd" + "lol.mleku.dev/chk" + "lol.mleku.dev/log" ) const ( @@ -877,6 +877,7 @@ func (a *Aggregator) fetchTimeWindow(client *ws.Client, relayURL string, window // Check if we've already seen this event if a.isEventSeen(eventID) { + ev.Free() continue } @@ -895,6 +896,9 @@ func (a *Aggregator) fetchTimeWindow(client *ws.Client, relayURL string, window if err = a.outputEvent(ev); chk.E(err) { log.E.F("failed to output event: %v", err) } + + // Free event after processing + ev.Free() case <-sub.EndOfStoredEvents: log.I.F("end of stored events for batch on relay %s", relayURL) batchComplete = true diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..071964d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "next.orly.dev", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/pkg/spider/directory.go b/pkg/spider/directory.go index 8420796..e809129 100644 --- a/pkg/spider/directory.go +++ b/pkg/spider/directory.go @@ -557,19 +557,31 @@ func (ds *DirectorySpider) fetchKindFromRelay(relayURL string, k uint16) ([]*eve // storeEvents saves events to the database and publishes new ones. func (ds *DirectorySpider) storeEvents(events []*event.E) (saved, duplicates int) { for _, ev := range events { - _, err := ds.db.SaveEvent(ds.ctx, ev) + savedEvent, err := ds.db.SaveEvent(ds.ctx, ev) if err != nil { if chk.T(err) { // Most errors are duplicates, which is expected duplicates++ } + // Free event on error + ev.Free() continue } - saved++ - - // Publish event to active subscribers - if ds.pub != nil { - go ds.pub.Deliver(ev) + if savedEvent { + saved++ + // Publish event to active subscribers + // Clone the event before delivering since Deliver sends it to channels + // and the receiver goroutines will handle freeing the cloned event + if ds.pub != nil { + cloned := ev.Clone() + go ds.pub.Deliver(cloned) + } + // Free the original event after cloning + ev.Free() + } else { + // Event was duplicate, free it + duplicates++ + ev.Free() } } return diff --git a/pkg/spider/spider.go b/pkg/spider/spider.go index 781f0ad..0785607 100644 --- a/pkg/spider/spider.go +++ b/pkg/spider/spider.go @@ -702,15 +702,25 @@ func (bs *BatchSubscription) handleEvents() { <-ticker.C // Save event to database - if _, err := bs.relay.spider.db.SaveEvent(bs.relay.ctx, ev); err != nil { - // Ignore duplicate events and other errors + saved, err := bs.relay.spider.db.SaveEvent(bs.relay.ctx, ev) + if err != nil { + // Ignore duplicate events and other errors, but free the event log.T.F("spider: failed to save event from %s: %v", bs.relay.url, err) - } else { + ev.Free() // event.E method + } else if saved { // Publish event if it was newly saved + // Clone the event before delivering since Deliver sends it to channels + // and the receiver goroutines will handle freeing the cloned event if bs.relay.spider.pub != nil { - go bs.relay.spider.pub.Deliver(ev) + cloned := ev.Clone() // event.E method + go bs.relay.spider.pub.Deliver(cloned) } + // Free the original event after cloning + ev.Free() // event.E method log.T.F("spider: saved event from %s", bs.relay.url) + } else { + // Event was duplicate, free it + ev.Free() // event.E method } } } @@ -851,15 +861,25 @@ func (rc *RelayConnection) performCatchup(sub *BatchSubscription, disconnectTime eventCount++ // Save event to database - if _, err := rc.spider.db.SaveEvent(rc.ctx, ev); err != nil { - // Silently ignore errors (mostly duplicates) - } else { + saved, err := rc.spider.db.SaveEvent(rc.ctx, ev) + if err != nil { + // Silently ignore errors (mostly duplicates), but free the event + ev.Free() + } else if saved { // Publish event if it was newly saved + // Clone the event before delivering since Deliver sends it to channels + // and the receiver goroutines will handle freeing the cloned event if rc.spider.pub != nil { - go rc.spider.pub.Deliver(ev) + cloned := ev.Clone() + go rc.spider.pub.Deliver(cloned) } + // Free the original event after cloning + ev.Free() log.T.F("spider: catch-up saved event %s from %s", hex.Enc(ev.ID[:]), rc.url) + } else { + // Event was duplicate, free it + ev.Free() } } } diff --git a/run-orly-docker.sh b/run-orly-docker.sh index f5c0e4f..5e3d5a3 100755 --- a/run-orly-docker.sh +++ b/run-orly-docker.sh @@ -36,10 +36,10 @@ echo "Starting ${CONTAINER_NAME}..." docker run -d \ --name ${CONTAINER_NAME} \ --restart always \ - -p 127.0.0.1:3334:3334 \ - -p 127.0.0.1:7777:7777 \ + -p 0.0.0.0:3334:3334 \ + -p 0.0.0.0:7777:7777 \ -v "${DATA_DIR}:/data" \ - --memory=4096m \ + --memory=6144m \ --cpus="2.0" \ --health-cmd="curl -f http://localhost:7777/ || exit 1" \ --health-interval=10s \ @@ -57,10 +57,10 @@ docker run -d \ -e ORLY_RELAY_URL=wss://orly-relay.imwald.eu \ -e ORLY_SPROCKET_ENABLED=false \ -e ORLY_DB_LOG_LEVEL=error \ - -e ORLY_DB_BLOCK_CACHE_MB=2048 \ - -e ORLY_DB_INDEX_CACHE_MB=1024 \ - -e ORLY_SERIAL_CACHE_PUBKEYS=500000 \ - -e ORLY_SERIAL_CACHE_EVENT_IDS=2000000 \ + -e ORLY_DB_BLOCK_CACHE_MB=512 \ + -e ORLY_DB_INDEX_CACHE_MB=256 \ + -e ORLY_SERIAL_CACHE_PUBKEYS=100000 \ + -e ORLY_SERIAL_CACHE_EVENT_IDS=500000 \ -e ORLY_DB_ZSTD_LEVEL=9 \ -e ORLY_GC_ENABLED=true \ -e ORLY_GC_BATCH_SIZE=5000 \ @@ -71,6 +71,10 @@ docker run -d \ -e ORLY_MAX_CONNECTIONS=1000 \ -e ORLY_MAX_EVENT_SIZE=65536 \ -e ORLY_MAX_SUBSCRIPTIONS=20 \ + -e ORLY_WEB_DISABLE=false \ + -e ORLY_WEB_DEV_PROXY_URL="" \ + -e ORLY_RATE_LIMIT_EMERGENCY_THRESHOLD=4.0 \ + -e ORLY_RATE_LIMIT_RECOVERY_THRESHOLD=3.0 \ ${IMAGE} echo "" diff --git a/scripts/update-embedded-web.sh b/scripts/update-embedded-web.sh index b21208a..f5b247b 100755 --- a/scripts/update-embedded-web.sh +++ b/scripts/update-embedded-web.sh @@ -3,8 +3,8 @@ # Build the embedded web UI and then install the Go binary. # # This script will: -# - Build the React app in app/web to app/web/dist using Bun (preferred), -# or fall back to npm/yarn/pnpm if Bun isn't available. +# - Build the Svelte app in app/web to app/web/dist using npm (preferred), +# or fall back to yarn/pnpm if npm isn't available. # - Run `go install` from the repository root so the binary picks up the new # embedded assets. # @@ -13,7 +13,7 @@ # # Requirements: # - Go 1.18+ installed (for `go install` and go:embed support) -# - Bun (https://bun.sh) recommended; alternatively Node.js with npm/yarn/pnpm +# - Node.js with npm (recommended); alternatively yarn or pnpm # set -euo pipefail @@ -32,16 +32,14 @@ fi # Choose a JS package runner JS_RUNNER="" -if command -v bun >/dev/null 2>&1; then - JS_RUNNER="bun" -elif command -v npm >/dev/null 2>&1; then +if command -v npm >/dev/null 2>&1; then JS_RUNNER="npm" elif command -v yarn >/dev/null 2>&1; then JS_RUNNER="yarn" elif command -v pnpm >/dev/null 2>&1; then JS_RUNNER="pnpm" else - err "No JavaScript package manager found. Install Bun (recommended) or npm/yarn/pnpm." + err "No JavaScript package manager found. Install npm (recommended), yarn, or pnpm." exit 1 fi @@ -51,11 +49,6 @@ log "Using JavaScript runner: ${JS_RUNNER}" log "Installing frontend dependencies..." pushd "${WEB_DIR}" >/dev/null case "${JS_RUNNER}" in - bun) - bun install - log "Building web app with Bun..." - bun run build - ;; npm) npm ci || npm install log "Building web app with npm..."