working on import API

app/server.go

@@ -1,6 +1,8 @@
 package app
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -1187,20 +1189,40 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
 
 	// Skip authentication and permission checks when ACL is "none" (open relay mode)
 	if acl.Registry.GetMode() != "none" {
-		// Try localhost authentication first
+		// Try localhost authentication first (uses NOSTR_PRIVATE_KEY env var, no AUTH header required)
 		pubkey, err = s.authenticateLocalhost(r)
 		if err != nil {
-			// Fall back to NIP-98 authentication
-			valid, pk, authErr := httpauth.CheckAuth(r)
-			if chk.E(authErr) || !valid {
-				errorMsg := "NIP-98 authentication validation failed"
-				if authErr != nil {
-					errorMsg = authErr.Error()
+			// If NOSTR_PRIVATE_KEY is set, allow the request without NIP-98 auth
+			// This is useful for containerized environments where requests come from within the container
+			if nsec := os.Getenv("NOSTR_PRIVATE_KEY"); nsec != "" {
+				// Try to get pubkey from NOSTR_PRIVATE_KEY even if not localhost
+				secretBytes, decodeErr := bech32encoding.NsecToBytes([]byte(nsec))
+				if decodeErr == nil {
+					signer, signerErr := p8k.New()
+					if signerErr == nil {
+						if initErr := signer.InitSec(secretBytes); initErr == nil {
+							pubkey = signer.Pub()
+							log.Printf("[HTTP API IMPORT] Authenticated using NOSTR_PRIVATE_KEY (no NIP-98 header required)")
+						}
+					}
 				}
-				http.Error(w, errorMsg, http.StatusUnauthorized)
-				return
 			}
-			pubkey = pk
+			
+			// If we still don't have a pubkey, try NIP-98 authentication
+			if pubkey == nil {
+				valid, pk, authErr := httpauth.CheckAuth(r)
+				if chk.E(authErr) || !valid {
+					errorMsg := "Authentication required. Use NOSTR_PRIVATE_KEY env var or NIP-98 Authorization header"
+					if authErr != nil {
+						errorMsg = authErr.Error()
+					}
+					http.Error(w, errorMsg, http.StatusUnauthorized)
+					return
+				}
+				pubkey = pk
+			}
+		} else {
+			log.Printf("[HTTP API IMPORT] Authenticated via localhost using NOSTR_PRIVATE_KEY")
 		}
 
 		// Check permissions - require write, admin, or owner level
@@ -1211,6 +1233,9 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
 			)
 			return
 		}
+	} else {
+		// Open relay mode - no authentication required
+		log.Printf("[HTTP API IMPORT] Open relay mode - no authentication required")
 	}
 
 	// Check if this is a folder import
@@ -1220,7 +1245,87 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Check if importing directly to a remote relay
+	targetURL := r.URL.Query().Get("target")
+	if targetURL != "" {
+		// Normalize WebSocket URL
+		if strings.HasPrefix(targetURL, "http://") {
+			targetURL = strings.Replace(targetURL, "http://", "ws://", 1)
+		} else if strings.HasPrefix(targetURL, "https://") {
+			targetURL = strings.Replace(targetURL, "https://", "wss://", 1)
+		} else if !strings.HasPrefix(targetURL, "ws://") && !strings.HasPrefix(targetURL, "wss://") {
+			targetURL = "wss://" + targetURL
+		}
+
+		// Get the file/body reader
+		var reader io.Reader
+		ct := r.Header.Get("Content-Type")
+		if strings.HasPrefix(ct, "multipart/form-data") {
+			if err := r.ParseMultipartForm(32 << 20); chk.E(err) {
+				http.Error(w, "Failed to parse form", http.StatusBadRequest)
+				return
+			}
+			file, _, err := r.FormFile("file")
+			if chk.E(err) {
+				http.Error(w, "Missing file", http.StatusBadRequest)
+				return
+			}
+			defer file.Close()
+			reader = file
+		} else {
+			if r.Body == nil {
+				http.Error(w, "Empty request body", http.StatusBadRequest)
+				return
+			}
+			reader = r.Body
+		}
+
+		// Check if we should wait for completion (default: yes, for sequential processing)
+		waitForCompletion := r.URL.Query().Get("async") != "true"
+
+		if waitForCompletion {
+			// Run synchronously - wait for completion before returning
+			// This allows the script to process files sequentially
+			if err := s.importToRemoteRelay(context.Background(), reader, targetURL); err != nil {
+				http.Error(w, fmt.Sprintf("Import to remote relay failed: %v", err), http.StatusInternalServerError)
+				return
+			}
+
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(`{"success": true, "message": "Import to remote relay completed", "target": "` + targetURL + `"}`))
+		} else {
+			// Run asynchronously in background
+			// IMPORTANT: Read the body into memory first, as r.Body will be closed when handler returns
+			bodyBytes, err := io.ReadAll(reader)
+			if err != nil {
+				log.Printf("ERROR: Failed to read request body for remote import: %v", err)
+				http.Error(w, fmt.Sprintf("Failed to read request body: %v", err), http.StatusInternalServerError)
+				return
+			}
+
+			log.Printf("Starting async import to remote relay %s (body size: %d bytes)", targetURL, len(bodyBytes))
+
+			go func() {
+				bodyReader := bytes.NewReader(bodyBytes)
+				if err := s.importToRemoteRelay(context.Background(), bodyReader, targetURL); err != nil {
+					log.Printf("ERROR: Import to remote relay %s failed: %v", targetURL, err)
+				}
+			}()
+
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusAccepted)
+			w.Write([]byte(`{"success": true, "message": "Import to remote relay started", "target": "` + targetURL + `"}`))
+		}
+		return
+	}
+
+	// Default: import to local database
+	// Check if we should wait for completion (default: yes, for reliability)
+	waitForCompletion := r.URL.Query().Get("async") != "true"
+
 	ct := r.Header.Get("Content-Type")
+	var reader io.Reader
 	if strings.HasPrefix(ct, "multipart/form-data") {
 		if err := r.ParseMultipartForm(32 << 20); chk.E(err) { // 32MB memory, rest to temp files
 			http.Error(w, "Failed to parse form", http.StatusBadRequest)
@@ -1232,18 +1337,230 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		defer file.Close()
-		s.DB.Import(file)
+		reader = file
 	} else {
 		if r.Body == nil {
 			http.Error(w, "Empty request body", http.StatusBadRequest)
 			return
 		}
-		s.DB.Import(r.Body)
+		reader = r.Body
 	}
 
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusAccepted)
-	w.Write([]byte(`{"success": true, "message": "Import started"}`))
+	if waitForCompletion {
+		// Run synchronously - wait for completion before returning
+		// This ensures the import is complete and we can return proper status
+		log.Printf("================================================")
+		log.Printf("[HTTP API IMPORT] Starting synchronous import")
+		log.Printf("[HTTP API IMPORT] This may take several minutes for large files")
+		log.Printf("================================================")
+
+		s.DB.Import(reader)
+
+		log.Printf("================================================")
+		log.Printf("[HTTP API IMPORT] Import completed successfully")
+		log.Printf("================================================")
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{"success": true, "message": "Import completed"}`))
+	} else {
+		// Run asynchronously in background
+		// IMPORTANT: Read the body into memory first, as r.Body will be closed when handler returns
+		bodyBytes, err := io.ReadAll(reader)
+		if err != nil {
+			log.Printf("ERROR: Failed to read request body for async import: %v", err)
+			http.Error(w, fmt.Sprintf("Failed to read request body: %v", err), http.StatusInternalServerError)
+			return
+		}
+
+		log.Printf("Starting async import (body size: %d bytes)", len(bodyBytes))
+
+		go func() {
+			bodyReader := bytes.NewReader(bodyBytes)
+			s.DB.Import(bodyReader)
+			log.Printf("Async import completed")
+		}()
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusAccepted)
+		w.Write([]byte(`{"success": true, "message": "Import started"}`))
+	}
+}
+
+// importToRemoteRelay reads JSONL from a reader and sends events directly to a remote relay
+func (s *Server) importToRemoteRelay(ctx context.Context, reader io.Reader, targetURL string) error {
+	log.Printf("Starting import to remote relay: %s", targetURL)
+
+	// Connect to remote relay
+	dialer := websocket.Dialer{
+		HandshakeTimeout: 30 * time.Second,
+	}
+
+	log.Printf("Attempting WebSocket connection to: %s", targetURL)
+	conn, _, err := dialer.DialContext(ctx, targetURL, http.Header{})
+	if err != nil {
+		log.Printf("ERROR: Failed to connect to remote relay %s: %v", targetURL, err)
+		return fmt.Errorf("failed to connect to relay: %w", err)
+	}
+	defer conn.Close()
+	log.Printf("Successfully connected to remote relay: %s", targetURL)
+
+	// Read JSONL line by line and send events
+	scanner := bufio.NewScanner(reader)
+	buf := make([]byte, 0, 64*1024)
+	scanner.Buffer(buf, 10*1024*1024) // 10MB max line size
+
+	var totalSent, totalAccepted, totalRejected int64
+	rejectedKinds := make(map[uint16]int)
+	rejectionReasons := make(map[string]int) // reason -> count
+	pendingEvents := make(map[string]uint16) // eventID -> kind (for matching OK responses)
+
+	// Start reading OK responses in background
+	type okResponse struct {
+		EventID  string
+		Accepted bool
+		Reason   string
+	}
+	okChan := make(chan okResponse, 10000)
+	doneReading := make(chan struct{})
+	go func() {
+		defer close(doneReading)
+		for {
+			var raw []interface{}
+			if err := conn.ReadJSON(&raw); err != nil {
+				return
+			}
+			if len(raw) >= 3 && raw[0] == "OK" {
+				eventID, _ := raw[1].(string)
+				accepted, _ := raw[2].(bool)
+				reason := ""
+				if len(raw) >= 4 {
+					reason, _ = raw[3].(string)
+				}
+				select {
+				case okChan <- okResponse{EventID: eventID, Accepted: accepted, Reason: reason}:
+				default:
+				}
+			}
+		}
+	}()
+
+	// Process events line by line
+	for scanner.Scan() {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		line := scanner.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+
+		// Parse event
+		ev := event.New()
+		if _, err := ev.Unmarshal(line); err != nil {
+			ev.Free()
+			continue
+		}
+
+		// Serialize event for sending
+		eventJSON := ev.Serialize()
+		var eventMap map[string]interface{}
+		if err := json.Unmarshal(eventJSON, &eventMap); err != nil {
+			ev.Free()
+			continue
+		}
+
+		// Send EVENT message
+		eventMsg := []interface{}{"EVENT", eventMap}
+		if err := conn.WriteJSON(eventMsg); err != nil {
+			ev.Free()
+			return fmt.Errorf("failed to send event: %w", err)
+		}
+
+		eventID := hex.Enc(ev.ID)
+		eventKind := ev.Kind
+		totalSent++
+
+		// Store pending event for OK response matching
+		pendingEvents[eventID] = eventKind
+
+		// Process any available OK responses (non-blocking)
+		for {
+			select {
+			case ok := <-okChan:
+				if kind, exists := pendingEvents[ok.EventID]; exists {
+					if ok.Accepted {
+						totalAccepted++
+					} else {
+						totalRejected++
+						rejectedKinds[kind]++
+						if ok.Reason != "" {
+							rejectionReasons[ok.Reason]++
+						}
+					}
+					delete(pendingEvents, ok.EventID)
+				}
+			default:
+				// No more responses available right now
+				goto continueLoop
+			}
+		}
+	continueLoop:
+
+		ev.Free()
+
+		// Progress logging every 1000 events
+		if totalSent%1000 == 0 {
+			log.Printf("Import to %s: %d sent (accepted: %d, rejected: %d, pending: %d)", targetURL, totalSent, totalAccepted, totalRejected, len(pendingEvents))
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("scanner error: %w", err)
+	}
+
+	// Wait for remaining OK responses (up to 30 seconds for large batches)
+	timeout := time.After(30 * time.Second)
+	waitStart := time.Now()
+	for len(pendingEvents) > 0 && time.Since(waitStart) < 30*time.Second {
+		select {
+		case ok := <-okChan:
+			if kind, exists := pendingEvents[ok.EventID]; exists {
+				if ok.Accepted {
+					totalAccepted++
+				} else {
+					totalRejected++
+					rejectedKinds[kind]++
+					if ok.Reason != "" {
+						rejectionReasons[ok.Reason]++
+					}
+				}
+				delete(pendingEvents, ok.EventID)
+			}
+		case <-timeout:
+			log.Printf("Timeout waiting for OK responses, %d events still pending", len(pendingEvents))
+			break
+		}
+	}
+
+	log.Printf("Import to %s completed: %d sent (accepted: %d, rejected: %d, no response: %d)", targetURL, totalSent, totalAccepted, totalRejected, len(pendingEvents))
+	if totalRejected > 0 {
+		log.Printf("Rejected events by kind: %v", rejectedKinds)
+		if len(rejectionReasons) > 0 {
+			log.Printf("Rejection reasons:")
+			for reason, count := range rejectionReasons {
+				log.Printf("  %s: %d", reason, count)
+			}
+		}
+	}
+	if len(pendingEvents) > 0 {
+		log.Printf("Warning: %d events sent but no OK response received (may have been accepted silently)", len(pendingEvents))
+	}
+
+	return nil
 }
 
 // handleImportFolder imports all .jsonl files from a folder

app/web/src/App.svelte

@@ -1844,6 +1844,17 @@
     $: filteredBaseTabs = baseTabs.filter((tab) => {
         const currentRole = currentEffectiveRole;
 
+        // Import tab: allow if ACL is "none" (open relay) or user has write/admin/owner permissions
+        if (tab.id === "import" && tab.requiresAdmin) {
+            if (aclMode === "none") {
+                return true; // Open relay, allow import
+            }
+            if (isLoggedIn && (currentRole === "write" || currentRole === "admin" || currentRole === "owner")) {
+                return true; // User has required permissions
+            }
+            return false; // Hide if no access
+        }
+
         if (
             tab.requiresAdmin &&
             (!isLoggedIn ||
@@ -2770,6 +2781,9 @@
     }
 
     function handleScroll(event) {
+        if (!event || !event.target) {
+            return; // Event or target is null, skip handling
+        }
         const { scrollTop, scrollHeight, clientHeight } = event.target;
         const threshold = 100; // Load more when 100px from bottom
 

build-image-v0.48.10.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-# Build script for next-orly v0.48.10 Docker image
-
-set -e
-
-# Check for Docker
-DOCKER_CMD=""
-if command -v docker >/dev/null 2>&1; then
-    DOCKER_CMD="docker"
-elif command -v podman >/dev/null 2>&1; then
-    DOCKER_CMD="podman"
-elif command -v docker.io >/dev/null 2>&1; then
-    DOCKER_CMD="docker.io"
-else
-    echo "Error: Docker is not installed or not in PATH."
-    echo ""
-    echo "To install Docker on Ubuntu/Debian:"
-    echo "  sudo apt install docker.io"
-    echo "  sudo systemctl enable --now docker"
-    echo "  sudo usermod -aG docker $USER"
-    echo "  # Then log out and back in"
-    echo ""
-    echo "Or install Podman (Docker alternative):"
-    echo "  sudo apt install podman"
-    echo ""
-    exit 1
-fi
-
-# Check if we're on the correct tag
-CURRENT_TAG=$(git describe --tags --exact-match HEAD 2>/dev/null || echo "")
-if [ "$CURRENT_TAG" != "v0.48.10" ]; then
-    echo "Checking out v0.48.10..."
-    git checkout v0.48.10
-fi
-
-# Check if app/web/dist exists (web UI already built)
-if [ -d "app/web/dist" ]; then
-    echo "Web UI already built, using existing Dockerfile..."
-    DOCKERFILE="Dockerfile"
-else
-    echo "Web UI not found, using Dockerfile.with-web (will build web UI in Docker)..."
-    DOCKERFILE="Dockerfile.with-web"
-fi
-
-# Build the Docker image with both version and latest tags
-echo "Building Docker image silberengel/next-orly:v0.48.10 using $DOCKER_CMD..."
-$DOCKER_CMD build -t silberengel/next-orly:v0.48.10 -t silberengel/next-orly:latest -f "$DOCKERFILE" .
-
-echo ""
-echo "Build complete! Image tags:"
-echo "  - silberengel/next-orly:v0.48.10"
-echo "  - silberengel/next-orly:latest"
-echo ""
-echo "To push to Docker Hub:"
-echo "  $DOCKER_CMD push silberengel/next-orly:v0.48.10"
-echo "  $DOCKER_CMD push silberengel/next-orly:latest"
-echo ""
-echo "To run with Docker Compose:"
-echo "  docker compose -f docker-compose-orly.yml up -d"
-echo "  docker compose -f docker-compose-orly.yml logs -f"
-echo "  docker compose -f docker-compose-orly.yml down"

build-image.sh

@@ -1,8 +1,41 @@
 #!/bin/bash
-# Build script for next-orly v0.58.5 Docker image
+# Build script for next-orly Docker image (auto-detects version)
 
 set -e
 
+# Auto-detect version from git tag, version file, or use default
+detect_version() {
+    # Try git tag first (most reliable)
+    VERSION=$(git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//' || echo "")
+    
+    # Fallback to version file
+    if [ -z "$VERSION" ] && [ -f "pkg/version/version" ]; then
+        VERSION=$(cat pkg/version/version | tr -d ' \n' | sed 's/^v//')
+    fi
+    
+    # Fallback to latest git tag
+    if [ -z "$VERSION" ]; then
+        VERSION=$(git tag | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 | sed 's/^v//' || echo "")
+    fi
+    
+    # Last resort: use default or prompt
+    if [ -z "$VERSION" ]; then
+        echo "Warning: Could not auto-detect version. Using default v0.58.10"
+        VERSION="0.58.10"
+    fi
+    
+    # Ensure it starts with 'v' for consistency
+    if [[ ! "$VERSION" =~ ^v ]]; then
+        VERSION="v${VERSION}"
+    fi
+    
+    echo "$VERSION"
+}
+
+VERSION=$(detect_version)
+echo "Detected version: $VERSION"
+echo ""
+
 # Check for Docker
 DOCKER_CMD=""
 if command -v docker >/dev/null 2>&1; then
@@ -26,11 +59,12 @@ else
     exit 1
 fi
 
-# Check if we're on the correct tag
+# Check if we're on the correct tag (informational only)
 CURRENT_TAG=$(git describe --tags --exact-match HEAD 2>/dev/null || echo "")
-if [ "$CURRENT_TAG" != "v0.58.5" ]; then
-    echo "Warning: Not on v0.58.5 tag (current: $CURRENT_TAG)"
-    echo "Continuing anyway..."
+if [ -n "$CURRENT_TAG" ] && [ "$CURRENT_TAG" != "$VERSION" ]; then
+    echo "Info: Current git tag is $CURRENT_TAG, but building version $VERSION"
+    echo "Continuing with version $VERSION..."
+    echo ""
 fi
 
 # Always rebuild web UI to ensure latest changes are included
@@ -140,38 +174,39 @@ trap restore_gomod EXIT
 
 # Build the Docker image with both version and latest tags
 # Local nostr clone (if found) is already copied into build context
-echo "Building Docker image silberengel/next-orly:v0.58.5 using $DOCKER_CMD..."
+IMAGE_NAME="silberengel/next-orly"
+echo "Building Docker image ${IMAGE_NAME}:${VERSION} using $DOCKER_CMD..."
 if [ "$DOCKER_CMD" = "docker" ]; then
     # Try using host network mode first (uses host DNS) - best for DNS issues
     if docker build --help 2>/dev/null | grep -q "\-\-network"; then
         echo "Using host network mode for better DNS resolution..."
-        $DOCKER_CMD build --network=host -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" .
+        $DOCKER_CMD build --network=host -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" .
     elif docker build --help 2>/dev/null | grep -q "\-\-dns"; then
         # Fallback to DNS configuration
         echo "Using DNS configuration (8.8.8.8, 8.8.4.4)..."
-        $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" .
+        $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" .
     else
         # Last resort - no DNS config
         echo "Warning: No DNS configuration available, build may fail..."
-        $DOCKER_CMD build -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" .
+        $DOCKER_CMD build -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" .
     fi
 else
     # buildx or other - try DNS if available
     if docker buildx build --help 2>/dev/null | grep -q "\-\-dns"; then
-        $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" .
+        $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" .
     else
-        $DOCKER_CMD build -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" .
+        $DOCKER_CMD build -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" .
     fi
 fi
 
 echo ""
 echo "Build complete! Image tags:"
-echo "  - silberengel/next-orly:v0.58.5"
-echo "  - silberengel/next-orly:latest"
+echo "  - ${IMAGE_NAME}:${VERSION}"
+echo "  - ${IMAGE_NAME}:latest"
 echo ""
 echo "To push to Docker Hub:"
-echo "  $DOCKER_CMD push silberengel/next-orly:v0.58.5"
-echo "  $DOCKER_CMD push silberengel/next-orly:latest"
+echo "  $DOCKER_CMD push ${IMAGE_NAME}:${VERSION}"
+echo "  $DOCKER_CMD push ${IMAGE_NAME}:latest"
 echo ""
 echo "Or run this script with --push to build and push automatically:"
 echo "  $0 --push"
@@ -180,8 +215,8 @@ echo "  $0 --push"
 if [ "$1" == "--push" ]; then
     echo ""
     echo "Pushing images to Docker Hub..."
-    $DOCKER_CMD push silberengel/next-orly:v0.58.5
-    $DOCKER_CMD push silberengel/next-orly:latest
+    $DOCKER_CMD push "${IMAGE_NAME}:${VERSION}"
+    $DOCKER_CMD push "${IMAGE_NAME}:latest"
     echo ""
     echo "✅ Images pushed successfully!"
 fi

go.mod

@@ -269,4 +269,4 @@ require (
 )
 
 retract v1.0.3
-replace git.mleku.dev/mleku/nostr => ../nostr
+replace git.mleku.dev/mleku/nostr => ./docker-build-context/nostr

pkg/database/import_utils.go

@@ -6,15 +6,16 @@ package database
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"runtime/debug"
 	"strings"
 	"time"
 
+	"git.mleku.dev/mleku/nostr/encoders/event"
 	"lol.mleku.dev/chk"
 	"lol.mleku.dev/log"
-	"git.mleku.dev/mleku/nostr/encoders/event"
 )
 
 const maxLen = 500000000
@@ -22,54 +23,89 @@ const maxLen = 500000000
 // ImportEventsFromReader imports events from an io.Reader containing JSONL data
 func (d *D) ImportEventsFromReader(ctx context.Context, rr io.Reader) error {
 	startTime := time.Now()
-	log.I.F("import: starting import operation")
+	log.I.F("================================================")
+	log.I.F("[HTTP API IMPORT] Starting import operation")
+	log.I.F("[HTTP API IMPORT] State: Initializing - preparing to receive file data")
+	log.I.F("================================================")
 
 	// store to disk so we can return fast
 	tmpPath := os.TempDir() + string(os.PathSeparator) + "orly"
 	os.MkdirAll(tmpPath, 0700)
 	tmp, err := os.CreateTemp(tmpPath, "")
 	if chk.E(err) {
+		log.E.F("[HTTP API IMPORT] ERROR: Failed to create temp file: %v", err)
+		log.E.F("[HTTP API IMPORT] Action: Check disk space and temp directory permissions")
 		return err
 	}
 	defer os.Remove(tmp.Name()) // Clean up temp file when done
 
-	log.I.F("import: buffering upload to %s", tmp.Name())
+	log.I.F("[HTTP API IMPORT] State: Buffering - receiving file data from client")
+	log.I.F("[HTTP API IMPORT] Temp file: %s", tmp.Name())
 	bufferStart := time.Now()
 	bytesBuffered, err := io.Copy(tmp, rr)
 	if chk.E(err) {
+		log.E.F("[HTTP API IMPORT] ERROR: Failed to buffer file data: %v", err)
+		log.E.F("[HTTP API IMPORT] Action: Check network connection and file integrity")
 		return err
 	}
 	bufferElapsed := time.Since(bufferStart)
-	log.I.F("import: buffered %.2f MB in %v (%.2f MB/sec)",
-		float64(bytesBuffered)/1024/1024, bufferElapsed.Round(time.Millisecond),
-		float64(bytesBuffered)/bufferElapsed.Seconds()/1024/1024)
+	fileSizeMB := float64(bytesBuffered) / 1024 / 1024
+	bufferSpeedMBps := fileSizeMB / bufferElapsed.Seconds()
+	log.I.F("[HTTP API IMPORT] State: Buffering complete")
+	log.I.F("[HTTP API IMPORT] File size: %.2f MB | Buffer time: %v | Speed: %.2f MB/sec",
+		fileSizeMB, bufferElapsed.Round(time.Millisecond), bufferSpeedMBps)
 
 	if _, err = tmp.Seek(0, 0); chk.E(err) {
+		log.E.F("[HTTP API IMPORT] ERROR: Failed to rewind temp file: %v", err)
 		return err
 	}
 
-	processErr := d.processJSONLEvents(ctx, tmp)
+	// Estimate number of events (rough estimate: ~500 bytes per event average)
+	estimatedEvents := int(float64(bytesBuffered) / 500)
+	log.I.F("[HTTP API IMPORT] State: Processing - reading and importing events")
+	log.I.F("[HTTP API IMPORT] Estimated events: ~%d (based on file size)", estimatedEvents)
+	log.I.F("[HTTP API IMPORT] Progress updates will appear every 5 seconds")
+	log.I.F("================================================")
+	processErr := d.processJSONLEvents(ctx, tmp, bytesBuffered)
 
 	totalElapsed := time.Since(startTime)
-	log.I.F("import: total operation time: %v", totalElapsed.Round(time.Millisecond))
+	log.I.F("================================================")
+	log.I.F("[HTTP API IMPORT] State: Completed")
+	log.I.F("[HTTP API IMPORT] Total time: %v", totalElapsed.Round(time.Millisecond))
+	if processErr != nil {
+		log.E.F("[HTTP API IMPORT] Result: FAILED")
+		log.E.F("[HTTP API IMPORT] Error: %v", processErr)
+		log.E.F("[HTTP API IMPORT] Action: Check error details above, verify file format is valid JSONL")
+		log.I.F("================================================")
+	} else {
+		log.I.F("[HTTP API IMPORT] Result: SUCCESS")
+		log.I.F("[HTTP API IMPORT] All events have been imported and are now available on the relay")
+		log.I.F("================================================")
+	}
 
 	return processErr
 }
 
 // ImportEventsFromStrings imports events from a slice of JSON strings with policy filtering
-func (d *D) ImportEventsFromStrings(ctx context.Context, eventJSONs []string, policyManager interface{ CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) }) error {
+func (d *D) ImportEventsFromStrings(ctx context.Context, eventJSONs []string, policyManager interface {
+	CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error)
+}) error {
 	// Create a reader from the string slice
 	reader := strings.NewReader(strings.Join(eventJSONs, "\n"))
-	return d.processJSONLEventsWithPolicy(ctx, reader, policyManager)
+	// Estimate total bytes (rough estimate)
+	totalBytes := int64(len(strings.Join(eventJSONs, "\n")))
+	return d.processJSONLEventsWithPolicy(ctx, reader, policyManager, totalBytes)
 }
 
 // processJSONLEvents processes JSONL events from a reader
-func (d *D) processJSONLEvents(ctx context.Context, rr io.Reader) error {
-	return d.processJSONLEventsWithPolicy(ctx, rr, nil)
+func (d *D) processJSONLEvents(ctx context.Context, rr io.Reader, totalBytes int64) error {
+	return d.processJSONLEventsWithPolicy(ctx, rr, nil, totalBytes)
 }
 
 // processJSONLEventsWithPolicy processes JSONL events from a reader with optional policy filtering
-func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, policyManager interface{ CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) }) error {
+func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, policyManager interface {
+	CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error)
+}, totalBytes int64) error {
 	// Create a scanner to read the buffer line by line
 	scan := bufio.NewScanner(rr)
 	scanBuf := make([]byte, maxLen)
@@ -80,17 +116,26 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
 	lastLogTime := startTime
 	const logInterval = 5 * time.Second
 
-	var count, total, skipped, policyRejected, unmarshalErrors, saveErrors int
+	var count, bytesRead, skipped, policyRejected, unmarshalErrors, saveErrors int
+	var kind3Count, largeEventCount int // Track noisy event types
+
+	// Track bytes read for progress percentage
+	bytesReadTotal := int64(0)
+
 	for scan.Scan() {
 		select {
 		case <-ctx.Done():
-			log.I.F("import: context closed after %d events", count)
+			log.I.F("[HTTP API IMPORT] State: Cancelled")
+			log.I.F("[HTTP API IMPORT] Processed %d events before cancellation", count)
 			return ctx.Err()
 		default:
 		}
 
 		b := scan.Bytes()
-		total += len(b) + 1
+		lineLen := len(b) + 1 // +1 for newline
+		bytesReadTotal += int64(lineLen)
+		bytesRead += lineLen
+
 		if len(b) < 1 {
 			skipped++
 			continue
@@ -101,28 +146,41 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
 			// return the pooled buffer on error
 			ev.Free()
 			unmarshalErrors++
-			log.W.F("failed to unmarshal event: %v", err)
+			// Only log unmarshal errors if they're frequent (not spam)
+			if unmarshalErrors <= 10 || unmarshalErrors%100 == 0 {
+				log.W.F("[HTTP API IMPORT] Failed to unmarshal event #%d: %v", unmarshalErrors, err)
+			}
 			continue
 		}
 
+		// Track event kinds for summary (but don't log each one)
+		if ev.Kind == 3 {
+			kind3Count++
+		}
+		if len(b) > 100000 { // Events larger than 100KB
+			largeEventCount++
+		}
+
 		// Apply policy checking if policy manager is provided
 		if policyManager != nil {
 			// For sync imports, we treat events as coming from system/trusted source
 			// Use nil pubkey and empty remote to indicate system-level import
 			allowed, policyErr := policyManager.CheckPolicy("write", ev, nil, "")
 			if policyErr != nil {
-				log.W.F("policy check failed for event %x: %v", ev.ID, policyErr)
 				ev.Free()
 				policyRejected++
+				// Only log policy errors if they're frequent (not spam)
+				if policyRejected <= 10 || policyRejected%100 == 0 {
+					log.W.F("[HTTP API IMPORT] Policy check failed for event %x: %v", ev.ID, policyErr)
+				}
 				continue
 			}
 			if !allowed {
-				log.D.F("policy rejected event %x during sync import", ev.ID)
 				ev.Free()
 				policyRejected++
+				// Don't log individual policy rejections (too noisy)
 				continue
 			}
-			log.D.F("policy allowed event %x during sync import", ev.ID)
 		}
 
 		// Apply rate limiting before write operation if limiter is configured
@@ -134,7 +192,10 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
 			// return the pooled buffer on error paths too
 			ev.Free()
 			saveErrors++
-			log.W.F("failed to save event: %v", err)
+			// Only log save errors if they're frequent (not spam)
+			if saveErrors <= 10 || saveErrors%100 == 0 {
+				log.W.F("[HTTP API IMPORT] Failed to save event #%d: %v", saveErrors, err)
+			}
 			continue
 		}
 
@@ -143,30 +204,99 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli
 		b = nil
 		count++
 
-		// Progress logging every logInterval
+		// Progress logging every logInterval with detailed information
 		if time.Since(lastLogTime) >= logInterval {
 			elapsed := time.Since(startTime)
 			eventsPerSec := float64(count) / elapsed.Seconds()
-			mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024
-			log.I.F("import: progress %d events saved, %.2f MB read, %.0f events/sec, %.2f MB/sec",
-				count, float64(total)/1024/1024, eventsPerSec, mbPerSec)
+			mbRead := float64(bytesRead) / 1024 / 1024
+			mbPerSec := mbRead / elapsed.Seconds()
+
+			// Calculate progress percentage if we have total bytes
+			progressPercent := ""
+			eta := ""
+			if totalBytes > 0 {
+				percent := float64(bytesReadTotal) / float64(totalBytes) * 100
+				if percent > 100 {
+					percent = 100
+				}
+				progressPercent = fmt.Sprintf(" | Progress: %.1f%%", percent)
+
+				// Estimate time remaining
+				if percent > 0 && percent < 100 && eventsPerSec > 0 {
+					remainingBytes := totalBytes - bytesReadTotal
+					remainingMB := float64(remainingBytes) / 1024 / 1024
+					if mbPerSec > 0 {
+						remainingSeconds := remainingMB / mbPerSec
+						eta = fmt.Sprintf(" | ETA: ~%v", time.Duration(remainingSeconds)*time.Second)
+					}
+				}
+			}
+
+			log.I.F("[HTTP API IMPORT] Progress: %d events saved | %.2f MB read | %.0f events/sec | %.2f MB/sec%s%s",
+				count, mbRead, eventsPerSec, mbPerSec, progressPercent, eta)
+
+			// Show error summary if there are errors
+			if unmarshalErrors > 0 || saveErrors > 0 || policyRejected > 0 {
+				log.W.F("[HTTP API IMPORT] Errors so far: %d unmarshal, %d save, %d policy rejected",
+					unmarshalErrors, saveErrors, policyRejected)
+			}
+
 			lastLogTime = time.Now()
 			debug.FreeOSMemory()
 		}
 	}
 
-	// Final summary
+	// Final summary with actionable information
 	elapsed := time.Since(startTime)
 	eventsPerSec := float64(count) / elapsed.Seconds()
-	mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024
-	log.I.F("import: completed - %d events saved, %.2f MB in %v (%.0f events/sec, %.2f MB/sec)",
-		count, float64(total)/1024/1024, elapsed.Round(time.Millisecond), eventsPerSec, mbPerSec)
-	if unmarshalErrors > 0 || saveErrors > 0 || policyRejected > 0 || skipped > 0 {
-		log.I.F("import: stats - %d unmarshal errors, %d save errors, %d policy rejected, %d skipped empty lines",
-			unmarshalErrors, saveErrors, policyRejected, skipped)
+	mbRead := float64(bytesRead) / 1024 / 1024
+	mbPerSec := mbRead / elapsed.Seconds()
+
+	log.I.F("[HTTP API IMPORT] ================================================")
+	log.I.F("[HTTP API IMPORT] Processing complete")
+	log.I.F("[HTTP API IMPORT] Events saved: %d | Data processed: %.2f MB | Time: %v",
+		count, mbRead, elapsed.Round(time.Millisecond))
+	log.I.F("[HTTP API IMPORT] Performance: %.0f events/sec | %.2f MB/sec", eventsPerSec, mbPerSec)
+
+	// Show statistics
+	if kind3Count > 0 || largeEventCount > 0 {
+		log.I.F("[HTTP API IMPORT] Event types: %d kind-3 (follow lists), %d large events (>100KB)",
+			kind3Count, largeEventCount)
+	}
+
+	// Show error summary with actionable information
+	hasErrors := false
+	if unmarshalErrors > 0 {
+		log.W.F("[HTTP API IMPORT] WARNING: %d events failed to unmarshal", unmarshalErrors)
+		log.W.F("[HTTP API IMPORT] Action: Check if file format is valid JSONL (one JSON object per line)")
+		hasErrors = true
+	}
+	if saveErrors > 0 {
+		log.W.F("[HTTP API IMPORT] WARNING: %d events failed to save", saveErrors)
+		log.W.F("[HTTP API IMPORT] Action: Check database disk space and permissions")
+		hasErrors = true
+	}
+	if policyRejected > 0 {
+		log.W.F("[HTTP API IMPORT] WARNING: %d events rejected by policy", policyRejected)
+		log.W.F("[HTTP API IMPORT] Action: Review ACL/policy settings if this is unexpected")
+		hasErrors = true
+	}
+	if skipped > 0 {
+		log.I.F("[HTTP API IMPORT] Info: %d empty lines skipped (this is normal)", skipped)
+	}
+
+	if !hasErrors {
+		log.I.F("[HTTP API IMPORT] Result: All events imported successfully")
+		log.I.F("[HTTP API IMPORT] Action: Events are now available on the relay")
+	} else {
+		log.W.F("[HTTP API IMPORT] Result: Import completed with errors (see warnings above)")
+		log.W.F("[HTTP API IMPORT] Action: Review error messages and take appropriate action")
 	}
+	log.I.F("[HTTP API IMPORT] ================================================")
 
 	if err := scan.Err(); err != nil {
+		log.E.F("[HTTP API IMPORT] ERROR: Scanner error: %v", err)
+		log.E.F("[HTTP API IMPORT] Action: Check file integrity and format")
 		return err
 	}
 

run-orly-docker.sh

@@ -1,11 +1,36 @@
 #!/bin/bash
 # Run Orly relay using docker run (alternative to docker-compose)
-# Optimized for large dataset imports (20GB+)
+# Optimized for large dataset imports (20GB+) with memory-conscious settings
+#
+# Memory optimizations applied:
+# - Reduced cache sizes (block: 256MB, index: 128MB)
+# - Lower serial caches (pubkeys: 50k, event IDs: 250k)
+# - Explicit rate limit target (1500MB) to prevent auto-detection issues
+# - Fixed emergency thresholds (1.167/0.833 instead of 4.0/3.0)
+# - Query result limit (256) to prevent unbounded memory usage
+# - Reduced max connections (500 instead of 1000)
 
 set -e
 
 CONTAINER_NAME="orly-relay"
-IMAGE="silberengel/next-orly:v0.58.5"
+# Auto-detect version or use default
+# Try multiple methods to detect version
+VERSION=""
+if command -v git &> /dev/null && [ -d .git ]; then
+    VERSION=$(git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//' || echo "")
+fi
+if [ -z "$VERSION" ] && [ -f pkg/version/version ]; then
+    VERSION=$(cat pkg/version/version 2>/dev/null | tr -d ' \n' | sed 's/^v//' || echo "")
+fi
+# Fallback to default if still empty
+if [ -z "$VERSION" ]; then
+    VERSION="0.58.10"
+fi
+# Ensure version starts with 'v'
+if [[ ! "$VERSION" =~ ^v ]]; then
+    VERSION="v${VERSION}"
+fi
+IMAGE="silberengel/next-orly:${VERSION}"
 
 # Data directory on host filesystem (change this to your desired path)
 # Using bind mount instead of volume for better performance with large datasets
@@ -39,8 +64,9 @@ docker run -d \
     -p 0.0.0.0:3334:3334 \
     -p 0.0.0.0:7777:7777 \
     -v "${DATA_DIR}:/data" \
-    --memory=6144m \
+    --memory=4096m \
     --cpus="2.0" \
+    --oom-kill-disable=false \
     --health-cmd="curl -f http://localhost:7777/ || exit 1" \
     --health-interval=10s \
     --health-timeout=5s \
@@ -49,7 +75,8 @@ docker run -d \
     -e ORLY_DATA_DIR=/data \
     -e ORLY_LISTEN=0.0.0.0 \
     -e ORLY_PORT=7777 \
-    -e ORLY_LOG_LEVEL=Info \
+    -e ORLY_LOG_LEVEL=Warn \
+    -e NOSTR_PRIVATE_KEY="${NOSTR_PRIVATE_KEY:-}" \
     -e ORLY_ADMINS=npub1m4ny6hjqzepn4rxknuq94c2gpqzr29ufkkw7ttcxyak7v43n6vvsajc2jl,npub1v30tsz9vw6ylpz63g0a702nj3xa26t3m7p5us8f2y2sd8v6cnsvq465zjx,npub12umrfdjgvdxt45g0y3ghwcyfagssjrv5qlm3t6pu2aa5vydwdmwq8q0z04,npub18cddpua960qjy3wmw7y9gmzr4h3ajlrwq3k9jnmqzlxke4qkg6gqeyaztw \
     -e ORLY_OWNERS=npub1m4ny6hjqzepn4rxknuq94c2gpqzr29ufkkw7ttcxyak7v43n6vvsajc2jl,npub1v30tsz9vw6ylpz63g0a702nj3xa26t3m7p5us8f2y2sd8v6cnsvq465zjx,npub12umrfdjgvdxt45g0y3ghwcyfagssjrv5qlm3t6pu2aa5vydwdmwq8q0z04,npub18cddpua960qjy3wmw7y9gmzr4h3ajlrwq3k9jnmqzlxke4qkg6gqeyaztw \
     -e ORLY_ACL_MODE=follows \
@@ -57,10 +84,10 @@ docker run -d \
     -e ORLY_RELAY_URL=wss://orly-relay.imwald.eu \
     -e ORLY_SPROCKET_ENABLED=false \
     -e ORLY_DB_LOG_LEVEL=error \
-    -e ORLY_DB_BLOCK_CACHE_MB=512 \
-    -e ORLY_DB_INDEX_CACHE_MB=256 \
-    -e ORLY_SERIAL_CACHE_PUBKEYS=100000 \
-    -e ORLY_SERIAL_CACHE_EVENT_IDS=500000 \
+    -e ORLY_DB_BLOCK_CACHE_MB=256 \
+    -e ORLY_DB_INDEX_CACHE_MB=128 \
+    -e ORLY_SERIAL_CACHE_PUBKEYS=50000 \
+    -e ORLY_SERIAL_CACHE_EVENT_IDS=250000 \
     -e ORLY_DB_ZSTD_LEVEL=9 \
     -e ORLY_GC_ENABLED=true \
     -e ORLY_GC_BATCH_SIZE=5000 \
@@ -68,13 +95,21 @@ docker run -d \
     -e ORLY_BOOTSTRAP_RELAYS=wss://profiles.nostr1.com,wss://purplepag.es,wss://relay.nostr.band,wss://relay.damus.io \
     -e ORLY_SUBSCRIPTION_ENABLED=false \
     -e ORLY_MONTHLY_PRICE_SAT=0 \
-    -e ORLY_MAX_CONNECTIONS=1000 \
+    -e ORLY_MAX_CONNECTIONS=500 \
     -e ORLY_MAX_EVENT_SIZE=65536 \
     -e ORLY_MAX_SUBSCRIPTIONS=20 \
+    -e ORLY_QUERY_RESULT_LIMIT=256 \
     -e ORLY_WEB_DISABLE=false \
     -e ORLY_WEB_DEV_PROXY_URL="" \
-    -e ORLY_RATE_LIMIT_EMERGENCY_THRESHOLD=4.0 \
-    -e ORLY_RATE_LIMIT_RECOVERY_THRESHOLD=3.0 \
+    -e ORLY_RATE_LIMIT_ENABLED=true \
+    -e ORLY_RATE_LIMIT_TARGET_MB=1500 \
+    -e ORLY_RATE_LIMIT_WRITE_TARGET=0.70 \
+    -e ORLY_RATE_LIMIT_WRITE_KP=1.0 \
+    -e ORLY_RATE_LIMIT_WRITE_KI=0.2 \
+    -e ORLY_RATE_LIMIT_MAX_WRITE_MS=2000 \
+    -e ORLY_RATE_LIMIT_EMERGENCY_THRESHOLD=1.167 \
+    -e ORLY_RATE_LIMIT_RECOVERY_THRESHOLD=0.833 \
+    -e ORLY_RATE_LIMIT_EMERGENCY_MAX_MS=5000 \
     ${IMAGE}
 
 echo ""

scripts/app/web/dist/index.html

@@ -0,0 +1 @@
+<!-- Placeholder - will be replaced by Docker build -->

scripts/cleanup-stuck-imports.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Cleanup stuck import containers on remote server
+# Usage: ./scripts/cleanup-stuck-imports.sh [remote_host] [remote_user]
+
+REMOTE_HOST="${REMOTE_HOST:-${1:-217.154.126.125}}"
+REMOTE_USER="${REMOTE_USER:-${2:-root}}"
+
+echo "Cleaning up stuck import containers on ${REMOTE_USER}@${REMOTE_HOST}..."
+
+# Find and stop all containers running orly db import
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --filter 'ancestor=silberengel/next-orly:v0.58.10' --format '{{.ID}} {{.Command}}' | grep -E 'db import|sh -c.*orly db' | awk '{print \$1}'" | while read container_id; do
+    if [ -n "$container_id" ]; then
+        echo "Stopping container: $container_id"
+        ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker stop $container_id 2>/dev/null && docker rm $container_id 2>/dev/null" || true
+    fi
+done
+
+echo "Cleanup complete. Remaining orly containers:"
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps -a | grep orly || echo 'No orly containers found'"

scripts/create-test-event.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"git.mleku.dev/mleku/nostr/encoders/bech32encoding"
+	"git.mleku.dev/mleku/nostr/encoders/event"
+	"git.mleku.dev/mleku/nostr/encoders/kind"
+	"git.mleku.dev/mleku/nostr/encoders/tag"
+	"git.mleku.dev/mleku/nostr/interfaces/signer/p8k"
+)
+
+func main() {
+	// Get NOSTR_PRIVATE_KEY from environment
+	nsec := os.Getenv("NOSTR_PRIVATE_KEY")
+	if nsec == "" {
+		fmt.Fprintf(os.Stderr, "ERROR: NOSTR_PRIVATE_KEY environment variable not set\n")
+		os.Exit(1)
+	}
+
+	// Decode nsec to get private key bytes
+	secretBytes, err := bech32encoding.NsecToBytes([]byte(nsec))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: Failed to decode nsec: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Create signer and initialize with private key
+	signer, err := p8k.New()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: Failed to create signer: %v\n", err)
+		os.Exit(1)
+	}
+	if err = signer.InitSec(secretBytes); err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: Failed to initialize signer: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Create a test event (kind 1 - text note)
+	ev := event.New()
+	ev.CreatedAt = time.Now().Unix()
+	ev.Kind = kind.TextNote.K
+	ev.Content = []byte("Pre-flight test event for import verification")
+	ev.Tags = tag.NewS()
+
+	// Sign the event
+	if err := ev.Sign(signer); err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: Failed to sign event: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Serialize to JSON
+	eventJSON := ev.Serialize()
+
+	// Output the JSON event
+	fmt.Println(string(eventJSON))
+}

scripts/import-exports-remote.sh

@@ -0,0 +1,754 @@
+#!/bin/bash
+
+# Import JSONL files to remote Orly relay
+# This script copies files from local exports directory to remote server and imports them
+# Usage: ./scripts/import-exports-remote.sh [remote_host] [remote_user]
+
+set -e
+
+# Configuration
+REMOTE_HOST="${REMOTE_HOST:-${1:-217.154.126.125}}"
+REMOTE_USER="${REMOTE_USER:-${2:-root}}"
+REMOTE_TEMP_DIR="${REMOTE_TEMP_DIR:-/root/tmp/orly}"
+DOCKER_CONTAINER="${DOCKER_CONTAINER:-orly-relay}"
+# NOSTR_PRIVATE_KEY for localhost authentication (reads from environment, e.g., .bashrc)
+NOSTR_PRIVATE_KEY="${NOSTR_PRIVATE_KEY:-}"
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# Get script directory and calculate exports path
+# Script is in scripts/, exports are in ../scripts/exports relative to script
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPORTS_DIR="$(cd "$SCRIPT_DIR/../../scripts/exports" 2>/dev/null && pwd || echo "")"
+
+# IMPORTANT: This script NEVER deletes source files from EXPORTS_DIR
+# It only copies files to remote server and moves them to a "done" folder after successful import
+
+# Helper function to query an event ID from the relay
+# Returns 0 if event found, 1 if not found
+query_event_on_relay() {
+    local event_id="$1"
+    local relay_ws_http="ws://${REMOTE_HOST}:7777/"
+    local relay_ws_https="wss://${REMOTE_HOST}:7777/"
+    local event_found=0
+    
+    for relay_ws in "$relay_ws_http" "$relay_ws_https"; do
+        if command -v websocat >/dev/null 2>&1; then
+            local query_msg="[\"REQ\",\"verify-query-$(date +%s)\",{\"ids\":[\"${event_id}\"]}]"
+            local query_result=$(timeout 10 bash -c "echo '$query_msg' | websocat '$relay_ws' 2>&1" | head -20)
+            
+            if echo "$query_result" | grep -q "\"id\":\"${event_id}\""; then
+                return 0
+            fi
+        elif command -v python3 >/dev/null 2>&1; then
+            python3 <<PYEOF
+import json
+import sys
+import asyncio
+import ssl
+try:
+    import websockets
+except ImportError:
+    sys.exit(1)
+
+async def query_event(relay_url, event_id):
+    try:
+        ssl_context = None
+        if relay_url.startswith('wss://'):
+            ssl_context = ssl.create_default_context()
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+        
+        async with websockets.connect(relay_url, timeout=10, ssl=ssl_context) as ws:
+            # Read initial messages (AUTH, etc.) but don't return True for them
+            try:
+                initial_response = await asyncio.wait_for(ws.recv(), timeout=3.0)
+                # Just consume it, don't check it
+            except asyncio.TimeoutError:
+                pass
+            except Exception:
+                pass
+            
+            # Send REQ for the specific event ID
+            req_msg = ["REQ", f"verify-query-{int(__import__('time').time())}", {"ids": [event_id]}]
+            await ws.send(json.dumps(req_msg))
+            
+            # Wait for responses - we need to find the actual EVENT with matching ID
+            # Keep reading until we get EOSE or timeout
+            found_event = False
+            try:
+                while True:
+                    response = await asyncio.wait_for(ws.recv(), timeout=5.0)
+                    data = json.loads(response)
+                    if isinstance(data, list) and len(data) > 0:
+                        if data[0] == "EVENT" and len(data) > 2:
+                            event_data = data[2]
+                            if isinstance(event_data, dict) and event_data.get("id") == event_id:
+                                found_event = True
+                                break
+                        elif data[0] == "EOSE":
+                            # End of stored events - stop waiting
+                            break
+            except asyncio.TimeoutError:
+                pass
+            except Exception:
+                pass
+            
+            return found_event
+    except Exception:
+        return False
+
+relay_url = "${relay_ws}"
+event_id = "${event_id}"
+result = asyncio.run(query_event(relay_url, event_id))
+if result:
+    sys.exit(0)
+else:
+    sys.exit(1)
+PYEOF
+            if [ $? -eq 0 ]; then
+                return 0
+            fi
+        fi
+    done
+    
+    return 1
+}
+
+# Check if exports directory exists
+if [ -z "$EXPORTS_DIR" ] || [ ! -d "$EXPORTS_DIR" ]; then
+    echo -e "${RED}Error: Exports directory not found at: $SCRIPT_DIR/../scripts/exports${NC}"
+    echo "Expected path: $(cd "$SCRIPT_DIR/.." && pwd)/scripts/exports"
+    exit 1
+fi
+
+# Find all JSONL files
+jsonl_files=("$EXPORTS_DIR"/*.jsonl)
+
+# Check if any JSONL files exist
+if [ ! -e "${jsonl_files[0]}" ]; then
+    echo -e "${YELLOW}No JSONL files found in: $EXPORTS_DIR${NC}"
+    exit 0
+fi
+
+# Count files
+file_count=0
+for file in "${jsonl_files[@]}"; do
+    if [ -f "$file" ]; then
+        file_count=$((file_count + 1))
+    fi
+done
+
+if [ "$file_count" -eq 0 ]; then
+    echo -e "${YELLOW}No JSONL files found in: $EXPORTS_DIR${NC}"
+    exit 0
+fi
+
+echo -e "${BLUE}=== Import Exports to Remote Server ===${NC}"
+echo ""
+echo "Exports directory: $EXPORTS_DIR"
+echo "Remote server: ${REMOTE_USER}@${REMOTE_HOST}"
+echo "Remote temp directory: ${REMOTE_TEMP_DIR}"
+echo "Docker container: ${DOCKER_CONTAINER}"
+echo "Files found: $file_count"
+echo ""
+
+# Check if SSH is available
+if ! command -v ssh >/dev/null 2>&1; then
+    echo -e "${RED}Error: ssh command not found. Please install OpenSSH client.${NC}"
+    exit 1
+fi
+
+# Check if SCP is available
+if ! command -v scp >/dev/null 2>&1; then
+    echo -e "${RED}Error: scp command not found. Please install OpenSSH client.${NC}"
+    exit 1
+fi
+
+# Test SSH connection
+echo -e "${CYAN}Testing SSH connection to ${REMOTE_USER}@${REMOTE_HOST}...${NC}"
+if ! ssh -o ConnectTimeout=10 -o BatchMode=yes "${REMOTE_USER}@${REMOTE_HOST}" "echo 'Connection successful'" 2>/dev/null; then
+    echo -e "${YELLOW}Warning: SSH connection test failed. You may be prompted for password/key.${NC}"
+    echo "Continuing anyway..."
+fi
+echo ""
+
+# Check if Docker container exists on remote
+echo -e "${CYAN}Checking Docker container on remote server...${NC}"
+if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --format '{{.Names}}' | grep -q '^${DOCKER_CONTAINER}$'" 2>/dev/null; then
+    echo -e "${RED}Error: Docker container '${DOCKER_CONTAINER}' not found on remote server.${NC}"
+    echo "Available containers:"
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --format '{{.Names}}'" 2>/dev/null || true
+    exit 1
+fi
+echo -e "${GREEN}✓ Docker container '${DOCKER_CONTAINER}' found${NC}"
+echo ""
+
+# Check if NOSTR_PRIVATE_KEY is available (local or in container)
+echo -e "${CYAN}Checking for NOSTR_PRIVATE_KEY...${NC}"
+container_has_key=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} sh -c 'echo \$NOSTR_PRIVATE_KEY'" 2>/dev/null | grep -v '^$' || echo "")
+
+if [ -n "$container_has_key" ]; then
+    echo -e "${GREEN}✓ NOSTR_PRIVATE_KEY: Found in container (will use for localhost authentication)${NC}"
+elif [ -n "$NOSTR_PRIVATE_KEY" ]; then
+    echo -e "${GREEN}✓ NOSTR_PRIVATE_KEY: Found in local environment${NC}"
+    echo -e "${CYAN}  Will pass it to container for localhost authentication${NC}"
+else
+    echo -e "${YELLOW}⚠ NOSTR_PRIVATE_KEY: Not found${NC}"
+    echo -e "${YELLOW}  Set it locally: export NOSTR_PRIVATE_KEY=nsec1your_key_here${NC}"
+    echo -e "${YELLOW}  Or add to container environment when running docker${NC}"
+fi
+echo ""
+
+# Create temp directory on remote (needed for pre-flight test)
+echo -e "${CYAN}Creating temporary directory on remote server...${NC}"
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p '${REMOTE_TEMP_DIR}'" || {
+    echo -e "${RED}Error: Failed to create temp directory on remote server${NC}"
+    exit 1
+}
+echo -e "${GREEN}✓ Temp directory created${NC}"
+echo ""
+
+# Pre-flight test: Send a test event and verify it appears on the relay
+# This runs BEFORE copying files to catch configuration issues early
+echo -e "${CYAN}=== Pre-flight Test: Verifying Import Works ===${NC}"
+echo "  Creating and sending a test event..."
+echo "  (This verifies configuration before copying large files)"
+
+# Determine which NOSTR_PRIVATE_KEY to use
+TEST_NOSTR_KEY=""
+if [ -n "$container_has_key" ]; then
+    TEST_NOSTR_KEY="$container_has_key"
+    echo "  Using NOSTR_PRIVATE_KEY from container for test event"
+elif [ -n "$NOSTR_PRIVATE_KEY" ]; then
+    TEST_NOSTR_KEY="$NOSTR_PRIVATE_KEY"
+    echo "  Using NOSTR_PRIVATE_KEY from local environment for test event"
+else
+    echo -e "${RED}✗ NOSTR_PRIVATE_KEY not found - cannot create signed test event${NC}"
+    echo -e "${RED}✗ Pre-flight test FAILED - Need NOSTR_PRIVATE_KEY for write permissions${NC}"
+    exit 1
+fi
+
+# Create a test event using Go
+TEST_EVENT_FILE=$(mktemp /tmp/orly-test-event-XXXXXX.jsonl)
+TEST_EVENT_ID=""
+
+# Get the script directory to find the Go program
+GO_PROGRAM="${SCRIPT_DIR}/create-test-event.go"
+
+# Compile and run the Go program
+if command -v go >/dev/null 2>&1; then
+    echo "  Creating signed test event with Go..."
+    
+    # Compile the Go program to a temporary binary
+    TEMP_BINARY=$(mktemp /tmp/orly-test-event-XXXXXX)
+    TEMP_ERR=$(mktemp /tmp/orly-test-event-err-XXXXXX)
+    
+    if ! go build -o "$TEMP_BINARY" "$GO_PROGRAM" 2>"$TEMP_ERR"; then
+        COMPILE_ERR=$(cat "$TEMP_ERR" 2>/dev/null || echo "")
+        echo -e "${RED}✗ Failed to compile test event generator${NC}"
+        if [ -n "$COMPILE_ERR" ]; then
+            echo "  Compile error: $COMPILE_ERR"
+        fi
+        rm -f "$TEST_EVENT_FILE" "$TEMP_BINARY" "$TEMP_ERR"
+        exit 1
+    fi
+    rm -f "$TEMP_ERR"
+    
+    # Run it with NOSTR_PRIVATE_KEY
+    TEST_EVENT_JSON=$(NOSTR_PRIVATE_KEY="$TEST_NOSTR_KEY" "$TEMP_BINARY" 2>"$TEMP_ERR")
+    EXIT_CODE=$?
+    ERROR_OUTPUT=$(cat "$TEMP_ERR" 2>/dev/null || echo "")
+    
+    # Clean up binary and error file
+    rm -f "$TEMP_BINARY" "$TEMP_ERR"
+    
+    if [ $EXIT_CODE -ne 0 ] || [ -z "$TEST_EVENT_JSON" ]; then
+        echo -e "${RED}✗ Failed to create signed test event${NC}"
+        if [ -n "$ERROR_OUTPUT" ]; then
+            echo "  Error: $ERROR_OUTPUT"
+        fi
+        rm -f "$TEST_EVENT_FILE"
+        exit 1
+    fi
+    
+    # Save the event JSON
+    echo "$TEST_EVENT_JSON" > "$TEST_EVENT_FILE"
+    
+    # Extract event ID
+    if command -v jq >/dev/null 2>&1; then
+        TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | jq -r '.id' 2>/dev/null || echo "")
+    elif command -v python3 >/dev/null 2>&1; then
+        TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | python3 -c "import sys, json; print(json.load(sys.stdin).get('id', ''))" 2>/dev/null || echo "")
+    else
+        # Fallback: extract ID using grep/sed
+        TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | grep -o '"id":"[^"]*"' | sed 's/"id":"\([^"]*\)"/\1/' | head -1)
+    fi
+else
+    echo -e "${RED}✗ Go compiler not found - cannot create signed test event${NC}"
+    echo -e "${YELLOW}  Install Go: https://golang.org/dl/${NC}"
+    exit 1
+fi
+
+if [ -z "$TEST_EVENT_ID" ] || [ ! -s "$TEST_EVENT_FILE" ]; then
+    echo -e "${RED}✗ Failed to create test event or extract event ID${NC}"
+    rm -f "$TEST_EVENT_FILE"
+    exit 1
+fi
+
+echo "  Test event ID: ${TEST_EVENT_ID:0:16}..."
+echo "  Uploading test event to container..."
+
+# Copy test event to remote server first, then into container
+REMOTE_TEMP_EVENT="/tmp/orly-test-event-$(date +%s).jsonl"
+if ! scp "$TEST_EVENT_FILE" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_TEMP_EVENT}" 2>&1; then
+    echo -e "${RED}✗ Failed to copy test event to remote server${NC}"
+    rm -f "$TEST_EVENT_FILE"
+    exit 1
+fi
+
+# Now copy from remote server into container
+if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker cp ${REMOTE_TEMP_EVENT} ${DOCKER_CONTAINER}:/tmp/test_event.jsonl" 2>&1; then
+    echo -e "${RED}✗ Failed to copy test event to container${NC}"
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_TEMP_EVENT}" 2>/dev/null || true
+    rm -f "$TEST_EVENT_FILE"
+    exit 1
+fi
+
+# Fix ownership and permissions so the orly user can read it
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u root ${DOCKER_CONTAINER} chown orly:orly /tmp/test_event.jsonl && docker exec -u root ${DOCKER_CONTAINER} chmod 644 /tmp/test_event.jsonl" 2>/dev/null || true
+
+# Clean up remote temp file
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_TEMP_EVENT}" 2>/dev/null || true
+
+# Send test event via import API
+echo "  Sending test event via import API..."
+REMOTE_RESPONSE_FILE="/tmp/orly-test-response-$(date +%s).txt"
+
+if [ -n "$container_has_key" ]; then
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time 25 --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/test_event.jsonl\" \"http://127.0.0.1:7777/api/import?async=true\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1
+    CURL_EXIT=$?
+elif [ -n "$NOSTR_PRIVATE_KEY" ]; then
+    escaped_key=$(echo "$NOSTR_PRIVATE_KEY" | sed "s/'/'\\\\''/g")
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} sh -c \"export NOSTR_PRIVATE_KEY=\\\"$escaped_key\\\" && curl --max-time 25 --connect-timeout 5 -s -w \\\"\\\\n%{http_code}\\\" -X POST -F \\\"file=@/tmp/test_event.jsonl\\\" \\\"http://127.0.0.1:7777/api/import?async=true\\\"\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1
+    CURL_EXIT=$?
+else
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time 25 --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/test_event.jsonl\" \"http://127.0.0.1:7777/api/import?async=true\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1
+    CURL_EXIT=$?
+fi
+
+# Retrieve the response file
+test_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_RESPONSE_FILE} 2>/dev/null" 2>/dev/null || echo "")
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_RESPONSE_FILE}" 2>/dev/null || true
+
+# Check if curl timed out or failed
+if [ $CURL_EXIT -eq 124 ]; then
+    echo -e "${RED}✗ Test upload timed out after 30 seconds${NC}"
+    rm -f "$TEST_EVENT_FILE"
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true
+    exit 1
+elif [ $CURL_EXIT -ne 0 ]; then
+    echo -e "${RED}✗ Test upload failed (exit code: $CURL_EXIT)${NC}"
+    rm -f "$TEST_EVENT_FILE"
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true
+    exit 1
+fi
+
+test_http_code=$(echo "$test_response" | tail -n1 | tr -d '[:space:]')
+test_body=$(echo "$test_response" | sed '$d')
+
+if [ "$test_http_code" -ne 200 ] && [ "$test_http_code" -ne 202 ]; then
+    echo -e "${RED}✗ Test upload failed (HTTP $test_http_code)${NC}"
+    rm -f "$TEST_EVENT_FILE"
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true
+    exit 1
+fi
+
+echo -e "${GREEN}✓ Test event uploaded successfully (HTTP $test_http_code)${NC}"
+echo "  Waiting 5 seconds for event to be processed..."
+sleep 5
+
+echo "  Querying relay for test event..."
+RELAY_WS_HTTP="ws://${REMOTE_HOST}:7777/"
+RELAY_WS_HTTPS="wss://${REMOTE_HOST}:7777/"
+RELAY_WS=""
+EVENT_FOUND=0
+
+# Try to query using websocat or Python
+for RELAY_WS in "$RELAY_WS_HTTP" "$RELAY_WS_HTTPS"; do
+    echo "  Trying ${RELAY_WS}..."
+    
+    if command -v websocat >/dev/null 2>&1; then
+        QUERY_MSG="[\"REQ\",\"test-query-$(date +%s)\",{\"ids\":[\"${TEST_EVENT_ID}\"]}]"
+        QUERY_RESULT=$(timeout 10 bash -c "echo '$QUERY_MSG' | websocat '$RELAY_WS' 2>&1" | head -20)
+        
+        if echo "$QUERY_RESULT" | grep -q "\"id\":\"${TEST_EVENT_ID}\""; then
+            EVENT_FOUND=1
+            break
+        fi
+    elif command -v python3 >/dev/null 2>&1; then
+        echo "  Using Python websockets to query..."
+        QUERY_RESULT=$(python3 <<PYEOF
+import json
+import sys
+import asyncio
+import ssl
+try:
+    import websockets
+except ImportError:
+    print("ERROR: websockets library not installed", file=sys.stderr)
+    sys.exit(1)
+
+async def query_event(relay_url, event_id):
+    try:
+        ssl_context = None
+        if relay_url.startswith('wss://'):
+            ssl_context = ssl.create_default_context()
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+        
+        async with websockets.connect(relay_url, timeout=10, ssl=ssl_context) as ws:
+            # Read initial messages (AUTH, etc.) but don't return True for them
+            try:
+                initial_response = await asyncio.wait_for(ws.recv(), timeout=3.0)
+                # Just consume it, don't check it
+            except asyncio.TimeoutError:
+                pass
+            except Exception:
+                pass
+            
+            # Send REQ for the specific event ID
+            req_msg = ["REQ", f"test-query-{int(__import__('time').time())}", {"ids": [event_id]}]
+            await ws.send(json.dumps(req_msg))
+            
+            # Wait for responses - we need to find the actual EVENT with matching ID
+            # Keep reading until we get EOSE or timeout
+            found_event = False
+            try:
+                while True:
+                    response = await asyncio.wait_for(ws.recv(), timeout=5.0)
+                    data = json.loads(response)
+                    if isinstance(data, list) and len(data) > 0:
+                        if data[0] == "EVENT" and len(data) > 2:
+                            event_data = data[2]
+                            if isinstance(event_data, dict) and event_data.get("id") == event_id:
+                                found_event = True
+                                break
+                        elif data[0] == "EOSE":
+                            # End of stored events - stop waiting
+                            break
+            except asyncio.TimeoutError:
+                pass
+            except Exception:
+                pass
+            
+            return found_event
+    except Exception:
+        return False
+
+relay_url = "${RELAY_WS}"
+event_id = "${TEST_EVENT_ID}"
+result = asyncio.run(query_event(relay_url, event_id))
+if result:
+    sys.exit(0)
+else:
+    sys.exit(1)
+PYEOF
+        )
+        QUERY_EXIT=$?
+        
+        if [ $QUERY_EXIT -eq 0 ]; then
+            EVENT_FOUND=1
+            break
+        fi
+    else
+        echo -e "${YELLOW}  Warning: Neither websocat nor Python websockets available${NC}"
+        break
+    fi
+done
+
+# Clean up test files
+rm -f "$TEST_EVENT_FILE"
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true
+
+if [ "$EVENT_FOUND" -eq 1 ]; then
+    echo -e "${GREEN}✓ Test event found on relay${NC}"
+    echo -e "${GREEN}✓ Pre-flight test PASSED - Import is working${NC}"
+else
+    echo -e "${RED}✗ Test event NOT found on relay${NC}"
+    echo -e "${RED}✗ Pre-flight test FAILED - Import is not working${NC}"
+    echo -e "${YELLOW}  Fix configuration issues before proceeding${NC}"
+    exit 1
+fi
+
+echo ""
+echo -e "${GREEN}✓ All pre-flight checks passed. Proceeding with file copy and import...${NC}"
+echo ""
+
+# Process files one at a time: copy, extract event ID, import, verify, move to done
+echo -e "${BLUE}=== Processing Files One at a Time ===${NC}"
+echo -e "${CYAN}Method: Using HTTP API import (synchronous, no container restart needed)${NC}"
+echo ""
+
+success_count=0
+fail_count=0
+current=0
+DONE_DIR="${REMOTE_TEMP_DIR}/done"
+
+# Ensure temp and done directories exist on remote
+ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p '${REMOTE_TEMP_DIR}' '${DONE_DIR}'" 2>/dev/null || true
+
+# Process each local file one at a time
+for local_file in "${jsonl_files[@]}"; do
+    if [ ! -f "$local_file" ]; then
+        continue
+    fi
+    
+    current=$((current + 1))
+    filename=$(basename "$local_file")
+    file_size=$(du -h "$local_file" | cut -f1)
+    remote_file="${REMOTE_TEMP_DIR}/${filename}"
+    
+    echo ""
+    echo -e "${BLUE}=== [$current/$file_count] Processing: $filename (${file_size}) ===${NC}"
+    echo ""
+    
+    # Step 1: Copy file to remote server
+    echo -e "${CYAN}[1/5] Copying file to remote server...${NC}"
+    if ! scp "$local_file" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_TEMP_DIR}/" 2>&1; then
+        echo -e "${RED}✗ Failed to copy: $filename${NC}"
+        fail_count=$((fail_count + 1))
+        echo -e "${YELLOW}  Skipping this file and continuing to next...${NC}"
+        continue
+    fi
+    echo -e "${GREEN}✓ Copied: $filename${NC}"
+    echo ""
+    
+    # Step 2: Extract event ID from the remote file
+    echo -e "${CYAN}[2/5] Extracting event ID from file...${NC}"
+    file_event_id=""
+    
+    # Extract first event ID from the file (look for "id":"..." pattern)
+    # Try multiple patterns to handle different JSON formats
+    file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -oE '\"id\"[[:space:]]*:[[:space:]]*\"[a-f0-9]{64}\"' | head -1 | sed -E 's/\"id\"[[:space:]]*:[[:space:]]*\"([^\"]+)\"/\1/'" || echo "")
+    
+    # If that didn't work, try simpler pattern
+    if [ -z "$file_event_id" ]; then
+        file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -o '\"id\":\"[a-f0-9]\{64\}\"' | head -1 | sed 's/\"id\":\"\([^\"]*\)\"/\1/'" || echo "")
+    fi
+    
+    # If still no ID, try to find any 64-char hex string that looks like an event ID
+    if [ -z "$file_event_id" ]; then
+        file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -oE '[a-f0-9]{64}' | head -1" || echo "")
+    fi
+    
+    if [ -n "$file_event_id" ]; then
+        echo -e "${GREEN}✓ Extracted event ID: ${file_event_id:0:16}...${NC}"
+    else
+        echo -e "${YELLOW}⚠ Could not extract event ID (will skip verification)${NC}"
+    fi
+    echo ""
+    
+    # Step 3: Import the file via HTTP API
+    echo -e "${CYAN}[3/5] Importing file via HTTP API...${NC}"
+    
+    # Check file size first
+    file_size_bytes=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "stat -c%s '${remote_file}' 2>/dev/null" || echo "0")
+    file_size_mb=$(echo "scale=2; $file_size_bytes / 1024 / 1024" | bc 2>/dev/null || echo "0")
+    echo "  File size: ${file_size_mb} MB"
+    
+    import_success=false
+    
+    # Copy file into container for HTTP API import
+    echo "  Copying file into container..."
+    if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker cp '${remote_file}' ${DOCKER_CONTAINER}:/tmp/" 2>&1; then
+        echo -e "${RED}✗ Failed to copy file into container${NC}"
+        fail_count=$((fail_count + 1))
+        echo -e "${YELLOW}  Skipping this file and continuing to next...${NC}"
+        continue
+    fi
+    
+    # Fix ownership and permissions
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u root ${DOCKER_CONTAINER} chown orly:orly /tmp/${filename} && docker exec -u root ${DOCKER_CONTAINER} chmod 644 /tmp/${filename}" 2>/dev/null || true
+    
+    # Calculate timeout based on file size
+    timeout_seconds=$((file_size_bytes / 1024 / 1024 * 30))  # ~30 seconds per MB
+    if [ $timeout_seconds -lt 300 ]; then
+        timeout_seconds=300  # Minimum 5 minutes
+    elif [ $timeout_seconds -gt 7200 ]; then
+        timeout_seconds=7200  # Maximum 2 hours
+    fi
+    echo "  Using timeout: ${timeout_seconds}s for ${file_size_mb} MB file"
+    echo ""
+    
+    # Start streaming container logs in background to show import progress
+    # Use prominent log prefix so it's easy to find in logs
+    LOG_PREFIX="[IMPORT ${filename}]"
+    echo -e "${CYAN}  Starting import via HTTP API (synchronous mode)...${NC}"
+    echo -e "${CYAN}  Streaming container logs with prefix: ${LOG_PREFIX}${NC}"
+    echo ""
+    
+    # Start log streaming in background
+    LOG_TAIL_PID=""
+    (ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker logs -f --tail 0 ${DOCKER_CONTAINER} 2>&1" | while IFS= read -r line; do
+        # Show all lines with [HTTP API IMPORT] prefix (our new prominent logging)
+        # Also show other import-related lines for context
+        if echo "$line" | grep -qiE "\[HTTP API IMPORT\]|import.*progress|import.*complete|import.*failed|import.*error"; then
+            echo -e "${CYAN}  ${LOG_PREFIX} $line${NC}" >&2
+        fi
+    done) &
+    LOG_TAIL_PID=$!
+    sleep 1  # Give log stream a moment to start
+    
+    # Get NOSTR_PRIVATE_KEY for authentication
+    container_has_key=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} sh -c 'echo \$NOSTR_PRIVATE_KEY'" 2>/dev/null | grep -v '^$' || echo "")
+    
+    REMOTE_IMPORT_RESPONSE="/tmp/orly-import-response-$(date +%s).txt"
+    
+    # Show start time
+    UPLOAD_START=$(date +%s)
+    echo "  [$(date +%H:%M:%S)] Starting HTTP API import..."
+    echo -e "${CYAN}  ${LOG_PREFIX} Import started at $(date)${NC}"
+    
+    # Run import via HTTP API (synchronous - waits for completion)
+    if [ -n "$container_has_key" ]; then
+        echo "  Using NOSTR_PRIVATE_KEY from container (localhost authentication)"
+        ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/${filename}\" \"http://127.0.0.1:7777/api/import\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1
+        CURL_EXIT=$?
+    elif [ -n "$NOSTR_PRIVATE_KEY" ]; then
+        echo "  Passing NOSTR_PRIVATE_KEY to container for localhost authentication"
+        escaped_key=$(echo "$NOSTR_PRIVATE_KEY" | sed "s/'/'\\\\''/g")
+        ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} sh -c \"export NOSTR_PRIVATE_KEY=\\\"$escaped_key\\\" && curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \\\"\\\\n%{http_code}\\\" -X POST -F \\\"file=@/tmp/${filename}\\\" \\\"http://127.0.0.1:7777/api/import\\\"\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1
+        CURL_EXIT=$?
+    else
+        echo -e "${YELLOW}  Warning: No NOSTR_PRIVATE_KEY available${NC}"
+        echo "  Trying without authentication (may fail if ACL requires auth)..."
+        ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/${filename}\" \"http://127.0.0.1:7777/api/import\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1
+        CURL_EXIT=$?
+    fi
+    
+    UPLOAD_END=$(date +%s)
+    UPLOAD_DURATION=$((UPLOAD_END - UPLOAD_START))
+    echo "  [$(date +%H:%M:%S)] HTTP request completed in ${UPLOAD_DURATION}s"
+    
+    # Stop log streaming
+    if [ -n "$LOG_TAIL_PID" ] && kill -0 "$LOG_TAIL_PID" 2>/dev/null; then
+        kill "$LOG_TAIL_PID" 2>/dev/null || true
+        wait "$LOG_TAIL_PID" 2>/dev/null || true
+    fi
+    
+    # Get response
+    import_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_IMPORT_RESPONSE} 2>/dev/null" 2>/dev/null || echo "")
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_IMPORT_RESPONSE}" 2>/dev/null || true
+    
+    # Clean up file from container
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f '/tmp/${filename}'" 2>/dev/null || true
+    
+    # Check if curl timed out
+    if [ $CURL_EXIT -eq 124 ]; then
+        echo -e "${RED}✗ Import timed out after ${timeout_seconds} seconds${NC}"
+        echo -e "${CYAN}  ${LOG_PREFIX} Import timed out${NC}"
+        fail_count=$((fail_count + 1))
+        echo -e "${YELLOW}  Skipping this file and continuing to next...${NC}"
+        continue
+    elif [ $CURL_EXIT -ne 0 ]; then
+        echo -e "${RED}✗ Import failed (exit code: $CURL_EXIT)${NC}"
+        echo -e "${CYAN}  ${LOG_PREFIX} Import failed with exit code $CURL_EXIT${NC}"
+        fail_count=$((fail_count + 1))
+        echo -e "${YELLOW}  Skipping this file and continuing to next...${NC}"
+        continue
+    fi
+    
+    # Extract HTTP status code and body
+    import_http_code=$(echo "$import_response" | tail -n1 | tr -d '[:space:]')
+    import_body=$(echo "$import_response" | sed '$d')
+    
+    # Check if import was successful
+    if [ -z "$import_http_code" ] || ! [[ "$import_http_code" =~ ^[0-9]+$ ]]; then
+        fail_count=$((fail_count + 1))
+        echo -e "${RED}✗ Failed to import: $filename (invalid HTTP response)${NC}"
+        echo -e "${CYAN}  ${LOG_PREFIX} Invalid HTTP response${NC}"
+    elif [ "$import_http_code" -eq 200 ]; then
+        if echo "$import_body" | grep -qi '"success":\s*true\|"message".*completed'; then
+            echo -e "${GREEN}✓ Imported: $filename (HTTP $import_http_code)${NC}"
+            echo -e "${CYAN}  ${LOG_PREFIX} Import completed successfully${NC}"
+            import_success=true
+            success_count=$((success_count + 1))
+        else
+            fail_count=$((fail_count + 1))
+            echo -e "${RED}✗ Failed to import: $filename (HTTP $import_http_code, but no success in response)${NC}"
+            echo -e "${CYAN}  ${LOG_PREFIX} Import response indicates failure${NC}"
+        fi
+    else
+        fail_count=$((fail_count + 1))
+        echo -e "${RED}✗ Failed to import: $filename (HTTP $import_http_code)${NC}"
+        echo -e "${CYAN}  ${LOG_PREFIX} Import failed with HTTP $import_http_code${NC}"
+        if [ -n "$import_body" ]; then
+            echo "  Response: $import_body"
+        fi
+    fi
+    
+    # Step 4: Verify import
+    if [ "$import_success" = true ]; then
+        echo ""
+        echo -e "${CYAN}[4/5] Verifying import...${NC}"
+        if [ -n "$file_event_id" ]; then
+            echo "  Verifying imported event exists on relay..."
+            sleep 3  # Give relay a moment to process the imported events
+            if query_event_on_relay "$file_event_id"; then
+                echo -e "${GREEN}✓ Verified: Event ${file_event_id:0:16}... from ${filename} found on relay${NC}"
+            else
+                echo -e "${YELLOW}⚠ Warning: Event ${file_event_id:0:16}... from ${filename} not yet found on relay (may need more time)${NC}"
+            fi
+        else
+            echo -e "${YELLOW}⚠ Skipping verification (no event ID extracted)${NC}"
+        fi
+    else
+        echo ""
+        echo -e "${CYAN}[4/5] Skipping verification (import failed)${NC}"
+    fi
+    echo ""
+    
+    # Step 5: Move to done folder (only if import was successful)
+    if [ "$import_success" = true ]; then
+        echo -e "${CYAN}[5/5] Moving file to done folder...${NC}"
+        if ssh "${REMOTE_USER}@${REMOTE_HOST}" "test -f '${remote_file}'" 2>/dev/null; then
+            if ssh "${REMOTE_USER}@${REMOTE_HOST}" "mv '${remote_file}' '${DONE_DIR}/${filename}'" 2>/dev/null; then
+                echo -e "${GREEN}✓ Moved: ${filename} to ${DONE_DIR}${NC}"
+            else
+                echo -e "${YELLOW}⚠ Warning: Could not move ${filename} to done folder${NC}"
+            fi
+        else
+            echo -e "${YELLOW}⚠ Warning: File ${filename} not found on remote (may have been moved already)${NC}"
+        fi
+    else
+        echo -e "${CYAN}[5/5] Skipping move to done (import failed - file remains in ${REMOTE_TEMP_DIR})${NC}"
+    fi
+    echo ""
+done
+
+# Summary
+echo ""
+echo -e "${BLUE}=== Import Summary ===${NC}"
+echo -e "${GREEN}Files succeeded: $success_count${NC}"
+if [ "$fail_count" -gt 0 ]; then
+    echo -e "${RED}Files failed: $fail_count${NC}"
+fi
+echo ""
+
+if [ "$fail_count" -eq 0 ]; then
+    echo -e "${GREEN}All files imported successfully!${NC}"
+    echo -e "${GREEN}All successful files have been moved to ${DONE_DIR}${NC}"
+    exit 0
+else
+    echo -e "${YELLOW}Some imports failed. Check the errors above.${NC}"
+    echo -e "${YELLOW}Failed files remain in ${REMOTE_TEMP_DIR} for retry${NC}"
+    echo -e "${GREEN}Successful files have been moved to ${DONE_DIR}${NC}"
+    exit 1
+fi