From e0a40925a9c9d9e8ec6993f0fc2bc357741e3d9b Mon Sep 17 00:00:00 2001 From: Silberengel Date: Sun, 8 Feb 2026 12:19:23 +0100 Subject: [PATCH] working on import API --- app/server.go | 347 ++++++++++- app/web/src/App.svelte | 14 + build-image-v0.48.10.sh | 61 -- build-image-v0.58.5.sh => build-image.sh | 69 ++- go.mod | 2 +- pkg/database/import_utils.go | 194 +++++- run-orly-docker.sh | 57 +- scripts/app/web/dist/index.html | 1 + scripts/cleanup-stuck-imports.sh | 20 + scripts/create-test-event.go | 59 ++ scripts/import-exports-remote.sh | 754 +++++++++++++++++++++++ 11 files changed, 1441 insertions(+), 137 deletions(-) delete mode 100755 build-image-v0.48.10.sh rename build-image-v0.58.5.sh => build-image.sh (72%) create mode 100644 scripts/app/web/dist/index.html create mode 100755 scripts/cleanup-stuck-imports.sh create mode 100644 scripts/create-test-event.go create mode 100755 scripts/import-exports-remote.sh diff --git a/app/server.go b/app/server.go index a1fa7c8..4a7bd92 100644 --- a/app/server.go +++ b/app/server.go @@ -1,6 +1,8 @@ package app import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" @@ -1187,20 +1189,40 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) { // Skip authentication and permission checks when ACL is "none" (open relay mode) if acl.Registry.GetMode() != "none" { - // Try localhost authentication first + // Try localhost authentication first (uses NOSTR_PRIVATE_KEY env var, no AUTH header required) pubkey, err = s.authenticateLocalhost(r) if err != nil { - // Fall back to NIP-98 authentication - valid, pk, authErr := httpauth.CheckAuth(r) - if chk.E(authErr) || !valid { - errorMsg := "NIP-98 authentication validation failed" - if authErr != nil { - errorMsg = authErr.Error() + // If NOSTR_PRIVATE_KEY is set, allow the request without NIP-98 auth + // This is useful for containerized environments where requests come from within the container + if nsec := os.Getenv("NOSTR_PRIVATE_KEY"); nsec != "" { + // Try to get pubkey from NOSTR_PRIVATE_KEY even if not localhost + secretBytes, decodeErr := bech32encoding.NsecToBytes([]byte(nsec)) + if decodeErr == nil { + signer, signerErr := p8k.New() + if signerErr == nil { + if initErr := signer.InitSec(secretBytes); initErr == nil { + pubkey = signer.Pub() + log.Printf("[HTTP API IMPORT] Authenticated using NOSTR_PRIVATE_KEY (no NIP-98 header required)") + } + } } - http.Error(w, errorMsg, http.StatusUnauthorized) - return } - pubkey = pk + + // If we still don't have a pubkey, try NIP-98 authentication + if pubkey == nil { + valid, pk, authErr := httpauth.CheckAuth(r) + if chk.E(authErr) || !valid { + errorMsg := "Authentication required. Use NOSTR_PRIVATE_KEY env var or NIP-98 Authorization header" + if authErr != nil { + errorMsg = authErr.Error() + } + http.Error(w, errorMsg, http.StatusUnauthorized) + return + } + pubkey = pk + } + } else { + log.Printf("[HTTP API IMPORT] Authenticated via localhost using NOSTR_PRIVATE_KEY") } // Check permissions - require write, admin, or owner level @@ -1211,6 +1233,9 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) { ) return } + } else { + // Open relay mode - no authentication required + log.Printf("[HTTP API IMPORT] Open relay mode - no authentication required") } // Check if this is a folder import @@ -1220,7 +1245,87 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) { return } + // Check if importing directly to a remote relay + targetURL := r.URL.Query().Get("target") + if targetURL != "" { + // Normalize WebSocket URL + if strings.HasPrefix(targetURL, "http://") { + targetURL = strings.Replace(targetURL, "http://", "ws://", 1) + } else if strings.HasPrefix(targetURL, "https://") { + targetURL = strings.Replace(targetURL, "https://", "wss://", 1) + } else if !strings.HasPrefix(targetURL, "ws://") && !strings.HasPrefix(targetURL, "wss://") { + targetURL = "wss://" + targetURL + } + + // Get the file/body reader + var reader io.Reader + ct := r.Header.Get("Content-Type") + if strings.HasPrefix(ct, "multipart/form-data") { + if err := r.ParseMultipartForm(32 << 20); chk.E(err) { + http.Error(w, "Failed to parse form", http.StatusBadRequest) + return + } + file, _, err := r.FormFile("file") + if chk.E(err) { + http.Error(w, "Missing file", http.StatusBadRequest) + return + } + defer file.Close() + reader = file + } else { + if r.Body == nil { + http.Error(w, "Empty request body", http.StatusBadRequest) + return + } + reader = r.Body + } + + // Check if we should wait for completion (default: yes, for sequential processing) + waitForCompletion := r.URL.Query().Get("async") != "true" + + if waitForCompletion { + // Run synchronously - wait for completion before returning + // This allows the script to process files sequentially + if err := s.importToRemoteRelay(context.Background(), reader, targetURL); err != nil { + http.Error(w, fmt.Sprintf("Import to remote relay failed: %v", err), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"success": true, "message": "Import to remote relay completed", "target": "` + targetURL + `"}`)) + } else { + // Run asynchronously in background + // IMPORTANT: Read the body into memory first, as r.Body will be closed when handler returns + bodyBytes, err := io.ReadAll(reader) + if err != nil { + log.Printf("ERROR: Failed to read request body for remote import: %v", err) + http.Error(w, fmt.Sprintf("Failed to read request body: %v", err), http.StatusInternalServerError) + return + } + + log.Printf("Starting async import to remote relay %s (body size: %d bytes)", targetURL, len(bodyBytes)) + + go func() { + bodyReader := bytes.NewReader(bodyBytes) + if err := s.importToRemoteRelay(context.Background(), bodyReader, targetURL); err != nil { + log.Printf("ERROR: Import to remote relay %s failed: %v", targetURL, err) + } + }() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + w.Write([]byte(`{"success": true, "message": "Import to remote relay started", "target": "` + targetURL + `"}`)) + } + return + } + + // Default: import to local database + // Check if we should wait for completion (default: yes, for reliability) + waitForCompletion := r.URL.Query().Get("async") != "true" + ct := r.Header.Get("Content-Type") + var reader io.Reader if strings.HasPrefix(ct, "multipart/form-data") { if err := r.ParseMultipartForm(32 << 20); chk.E(err) { // 32MB memory, rest to temp files http.Error(w, "Failed to parse form", http.StatusBadRequest) @@ -1232,18 +1337,230 @@ func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) { return } defer file.Close() - s.DB.Import(file) + reader = file } else { if r.Body == nil { http.Error(w, "Empty request body", http.StatusBadRequest) return } - s.DB.Import(r.Body) + reader = r.Body } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusAccepted) - w.Write([]byte(`{"success": true, "message": "Import started"}`)) + if waitForCompletion { + // Run synchronously - wait for completion before returning + // This ensures the import is complete and we can return proper status + log.Printf("================================================") + log.Printf("[HTTP API IMPORT] Starting synchronous import") + log.Printf("[HTTP API IMPORT] This may take several minutes for large files") + log.Printf("================================================") + + s.DB.Import(reader) + + log.Printf("================================================") + log.Printf("[HTTP API IMPORT] Import completed successfully") + log.Printf("================================================") + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"success": true, "message": "Import completed"}`)) + } else { + // Run asynchronously in background + // IMPORTANT: Read the body into memory first, as r.Body will be closed when handler returns + bodyBytes, err := io.ReadAll(reader) + if err != nil { + log.Printf("ERROR: Failed to read request body for async import: %v", err) + http.Error(w, fmt.Sprintf("Failed to read request body: %v", err), http.StatusInternalServerError) + return + } + + log.Printf("Starting async import (body size: %d bytes)", len(bodyBytes)) + + go func() { + bodyReader := bytes.NewReader(bodyBytes) + s.DB.Import(bodyReader) + log.Printf("Async import completed") + }() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + w.Write([]byte(`{"success": true, "message": "Import started"}`)) + } +} + +// importToRemoteRelay reads JSONL from a reader and sends events directly to a remote relay +func (s *Server) importToRemoteRelay(ctx context.Context, reader io.Reader, targetURL string) error { + log.Printf("Starting import to remote relay: %s", targetURL) + + // Connect to remote relay + dialer := websocket.Dialer{ + HandshakeTimeout: 30 * time.Second, + } + + log.Printf("Attempting WebSocket connection to: %s", targetURL) + conn, _, err := dialer.DialContext(ctx, targetURL, http.Header{}) + if err != nil { + log.Printf("ERROR: Failed to connect to remote relay %s: %v", targetURL, err) + return fmt.Errorf("failed to connect to relay: %w", err) + } + defer conn.Close() + log.Printf("Successfully connected to remote relay: %s", targetURL) + + // Read JSONL line by line and send events + scanner := bufio.NewScanner(reader) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 10*1024*1024) // 10MB max line size + + var totalSent, totalAccepted, totalRejected int64 + rejectedKinds := make(map[uint16]int) + rejectionReasons := make(map[string]int) // reason -> count + pendingEvents := make(map[string]uint16) // eventID -> kind (for matching OK responses) + + // Start reading OK responses in background + type okResponse struct { + EventID string + Accepted bool + Reason string + } + okChan := make(chan okResponse, 10000) + doneReading := make(chan struct{}) + go func() { + defer close(doneReading) + for { + var raw []interface{} + if err := conn.ReadJSON(&raw); err != nil { + return + } + if len(raw) >= 3 && raw[0] == "OK" { + eventID, _ := raw[1].(string) + accepted, _ := raw[2].(bool) + reason := "" + if len(raw) >= 4 { + reason, _ = raw[3].(string) + } + select { + case okChan <- okResponse{EventID: eventID, Accepted: accepted, Reason: reason}: + default: + } + } + } + }() + + // Process events line by line + for scanner.Scan() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + line := scanner.Bytes() + if len(line) == 0 { + continue + } + + // Parse event + ev := event.New() + if _, err := ev.Unmarshal(line); err != nil { + ev.Free() + continue + } + + // Serialize event for sending + eventJSON := ev.Serialize() + var eventMap map[string]interface{} + if err := json.Unmarshal(eventJSON, &eventMap); err != nil { + ev.Free() + continue + } + + // Send EVENT message + eventMsg := []interface{}{"EVENT", eventMap} + if err := conn.WriteJSON(eventMsg); err != nil { + ev.Free() + return fmt.Errorf("failed to send event: %w", err) + } + + eventID := hex.Enc(ev.ID) + eventKind := ev.Kind + totalSent++ + + // Store pending event for OK response matching + pendingEvents[eventID] = eventKind + + // Process any available OK responses (non-blocking) + for { + select { + case ok := <-okChan: + if kind, exists := pendingEvents[ok.EventID]; exists { + if ok.Accepted { + totalAccepted++ + } else { + totalRejected++ + rejectedKinds[kind]++ + if ok.Reason != "" { + rejectionReasons[ok.Reason]++ + } + } + delete(pendingEvents, ok.EventID) + } + default: + // No more responses available right now + goto continueLoop + } + } + continueLoop: + + ev.Free() + + // Progress logging every 1000 events + if totalSent%1000 == 0 { + log.Printf("Import to %s: %d sent (accepted: %d, rejected: %d, pending: %d)", targetURL, totalSent, totalAccepted, totalRejected, len(pendingEvents)) + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("scanner error: %w", err) + } + + // Wait for remaining OK responses (up to 30 seconds for large batches) + timeout := time.After(30 * time.Second) + waitStart := time.Now() + for len(pendingEvents) > 0 && time.Since(waitStart) < 30*time.Second { + select { + case ok := <-okChan: + if kind, exists := pendingEvents[ok.EventID]; exists { + if ok.Accepted { + totalAccepted++ + } else { + totalRejected++ + rejectedKinds[kind]++ + if ok.Reason != "" { + rejectionReasons[ok.Reason]++ + } + } + delete(pendingEvents, ok.EventID) + } + case <-timeout: + log.Printf("Timeout waiting for OK responses, %d events still pending", len(pendingEvents)) + break + } + } + + log.Printf("Import to %s completed: %d sent (accepted: %d, rejected: %d, no response: %d)", targetURL, totalSent, totalAccepted, totalRejected, len(pendingEvents)) + if totalRejected > 0 { + log.Printf("Rejected events by kind: %v", rejectedKinds) + if len(rejectionReasons) > 0 { + log.Printf("Rejection reasons:") + for reason, count := range rejectionReasons { + log.Printf(" %s: %d", reason, count) + } + } + } + if len(pendingEvents) > 0 { + log.Printf("Warning: %d events sent but no OK response received (may have been accepted silently)", len(pendingEvents)) + } + + return nil } // handleImportFolder imports all .jsonl files from a folder diff --git a/app/web/src/App.svelte b/app/web/src/App.svelte index 2cdf32f..8fb3b7b 100644 --- a/app/web/src/App.svelte +++ b/app/web/src/App.svelte @@ -1844,6 +1844,17 @@ $: filteredBaseTabs = baseTabs.filter((tab) => { const currentRole = currentEffectiveRole; + // Import tab: allow if ACL is "none" (open relay) or user has write/admin/owner permissions + if (tab.id === "import" && tab.requiresAdmin) { + if (aclMode === "none") { + return true; // Open relay, allow import + } + if (isLoggedIn && (currentRole === "write" || currentRole === "admin" || currentRole === "owner")) { + return true; // User has required permissions + } + return false; // Hide if no access + } + if ( tab.requiresAdmin && (!isLoggedIn || @@ -2770,6 +2781,9 @@ } function handleScroll(event) { + if (!event || !event.target) { + return; // Event or target is null, skip handling + } const { scrollTop, scrollHeight, clientHeight } = event.target; const threshold = 100; // Load more when 100px from bottom diff --git a/build-image-v0.48.10.sh b/build-image-v0.48.10.sh deleted file mode 100755 index 69dd0a3..0000000 --- a/build-image-v0.48.10.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# Build script for next-orly v0.48.10 Docker image - -set -e - -# Check for Docker -DOCKER_CMD="" -if command -v docker >/dev/null 2>&1; then - DOCKER_CMD="docker" -elif command -v podman >/dev/null 2>&1; then - DOCKER_CMD="podman" -elif command -v docker.io >/dev/null 2>&1; then - DOCKER_CMD="docker.io" -else - echo "Error: Docker is not installed or not in PATH." - echo "" - echo "To install Docker on Ubuntu/Debian:" - echo " sudo apt install docker.io" - echo " sudo systemctl enable --now docker" - echo " sudo usermod -aG docker $USER" - echo " # Then log out and back in" - echo "" - echo "Or install Podman (Docker alternative):" - echo " sudo apt install podman" - echo "" - exit 1 -fi - -# Check if we're on the correct tag -CURRENT_TAG=$(git describe --tags --exact-match HEAD 2>/dev/null || echo "") -if [ "$CURRENT_TAG" != "v0.48.10" ]; then - echo "Checking out v0.48.10..." - git checkout v0.48.10 -fi - -# Check if app/web/dist exists (web UI already built) -if [ -d "app/web/dist" ]; then - echo "Web UI already built, using existing Dockerfile..." - DOCKERFILE="Dockerfile" -else - echo "Web UI not found, using Dockerfile.with-web (will build web UI in Docker)..." - DOCKERFILE="Dockerfile.with-web" -fi - -# Build the Docker image with both version and latest tags -echo "Building Docker image silberengel/next-orly:v0.48.10 using $DOCKER_CMD..." -$DOCKER_CMD build -t silberengel/next-orly:v0.48.10 -t silberengel/next-orly:latest -f "$DOCKERFILE" . - -echo "" -echo "Build complete! Image tags:" -echo " - silberengel/next-orly:v0.48.10" -echo " - silberengel/next-orly:latest" -echo "" -echo "To push to Docker Hub:" -echo " $DOCKER_CMD push silberengel/next-orly:v0.48.10" -echo " $DOCKER_CMD push silberengel/next-orly:latest" -echo "" -echo "To run with Docker Compose:" -echo " docker compose -f docker-compose-orly.yml up -d" -echo " docker compose -f docker-compose-orly.yml logs -f" -echo " docker compose -f docker-compose-orly.yml down" diff --git a/build-image-v0.58.5.sh b/build-image.sh similarity index 72% rename from build-image-v0.58.5.sh rename to build-image.sh index b37140b..ef2e5fd 100755 --- a/build-image-v0.58.5.sh +++ b/build-image.sh @@ -1,8 +1,41 @@ #!/bin/bash -# Build script for next-orly v0.58.5 Docker image +# Build script for next-orly Docker image (auto-detects version) set -e +# Auto-detect version from git tag, version file, or use default +detect_version() { + # Try git tag first (most reliable) + VERSION=$(git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//' || echo "") + + # Fallback to version file + if [ -z "$VERSION" ] && [ -f "pkg/version/version" ]; then + VERSION=$(cat pkg/version/version | tr -d ' \n' | sed 's/^v//') + fi + + # Fallback to latest git tag + if [ -z "$VERSION" ]; then + VERSION=$(git tag | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 | sed 's/^v//' || echo "") + fi + + # Last resort: use default or prompt + if [ -z "$VERSION" ]; then + echo "Warning: Could not auto-detect version. Using default v0.58.10" + VERSION="0.58.10" + fi + + # Ensure it starts with 'v' for consistency + if [[ ! "$VERSION" =~ ^v ]]; then + VERSION="v${VERSION}" + fi + + echo "$VERSION" +} + +VERSION=$(detect_version) +echo "Detected version: $VERSION" +echo "" + # Check for Docker DOCKER_CMD="" if command -v docker >/dev/null 2>&1; then @@ -26,11 +59,12 @@ else exit 1 fi -# Check if we're on the correct tag +# Check if we're on the correct tag (informational only) CURRENT_TAG=$(git describe --tags --exact-match HEAD 2>/dev/null || echo "") -if [ "$CURRENT_TAG" != "v0.58.5" ]; then - echo "Warning: Not on v0.58.5 tag (current: $CURRENT_TAG)" - echo "Continuing anyway..." +if [ -n "$CURRENT_TAG" ] && [ "$CURRENT_TAG" != "$VERSION" ]; then + echo "Info: Current git tag is $CURRENT_TAG, but building version $VERSION" + echo "Continuing with version $VERSION..." + echo "" fi # Always rebuild web UI to ensure latest changes are included @@ -140,38 +174,39 @@ trap restore_gomod EXIT # Build the Docker image with both version and latest tags # Local nostr clone (if found) is already copied into build context -echo "Building Docker image silberengel/next-orly:v0.58.5 using $DOCKER_CMD..." +IMAGE_NAME="silberengel/next-orly" +echo "Building Docker image ${IMAGE_NAME}:${VERSION} using $DOCKER_CMD..." if [ "$DOCKER_CMD" = "docker" ]; then # Try using host network mode first (uses host DNS) - best for DNS issues if docker build --help 2>/dev/null | grep -q "\-\-network"; then echo "Using host network mode for better DNS resolution..." - $DOCKER_CMD build --network=host -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" . + $DOCKER_CMD build --network=host -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" . elif docker build --help 2>/dev/null | grep -q "\-\-dns"; then # Fallback to DNS configuration echo "Using DNS configuration (8.8.8.8, 8.8.4.4)..." - $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" . + $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" . else # Last resort - no DNS config echo "Warning: No DNS configuration available, build may fail..." - $DOCKER_CMD build -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" . + $DOCKER_CMD build -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" . fi else # buildx or other - try DNS if available if docker buildx build --help 2>/dev/null | grep -q "\-\-dns"; then - $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" . + $DOCKER_CMD build --dns 8.8.8.8 --dns 8.8.4.4 -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" . else - $DOCKER_CMD build -t silberengel/next-orly:v0.58.5 -t silberengel/next-orly:latest -f "$DOCKERFILE" . + $DOCKER_CMD build -t "${IMAGE_NAME}:${VERSION}" -t "${IMAGE_NAME}:latest" -f "$DOCKERFILE" . fi fi echo "" echo "Build complete! Image tags:" -echo " - silberengel/next-orly:v0.58.5" -echo " - silberengel/next-orly:latest" +echo " - ${IMAGE_NAME}:${VERSION}" +echo " - ${IMAGE_NAME}:latest" echo "" echo "To push to Docker Hub:" -echo " $DOCKER_CMD push silberengel/next-orly:v0.58.5" -echo " $DOCKER_CMD push silberengel/next-orly:latest" +echo " $DOCKER_CMD push ${IMAGE_NAME}:${VERSION}" +echo " $DOCKER_CMD push ${IMAGE_NAME}:latest" echo "" echo "Or run this script with --push to build and push automatically:" echo " $0 --push" @@ -180,8 +215,8 @@ echo " $0 --push" if [ "$1" == "--push" ]; then echo "" echo "Pushing images to Docker Hub..." - $DOCKER_CMD push silberengel/next-orly:v0.58.5 - $DOCKER_CMD push silberengel/next-orly:latest + $DOCKER_CMD push "${IMAGE_NAME}:${VERSION}" + $DOCKER_CMD push "${IMAGE_NAME}:latest" echo "" echo "✅ Images pushed successfully!" fi diff --git a/go.mod b/go.mod index 5c27bbe..4f3ab9d 100644 --- a/go.mod +++ b/go.mod @@ -269,4 +269,4 @@ require ( ) retract v1.0.3 -replace git.mleku.dev/mleku/nostr => ../nostr +replace git.mleku.dev/mleku/nostr => ./docker-build-context/nostr diff --git a/pkg/database/import_utils.go b/pkg/database/import_utils.go index f600769..3872794 100644 --- a/pkg/database/import_utils.go +++ b/pkg/database/import_utils.go @@ -6,15 +6,16 @@ package database import ( "bufio" "context" + "fmt" "io" "os" "runtime/debug" "strings" "time" + "git.mleku.dev/mleku/nostr/encoders/event" "lol.mleku.dev/chk" "lol.mleku.dev/log" - "git.mleku.dev/mleku/nostr/encoders/event" ) const maxLen = 500000000 @@ -22,54 +23,89 @@ const maxLen = 500000000 // ImportEventsFromReader imports events from an io.Reader containing JSONL data func (d *D) ImportEventsFromReader(ctx context.Context, rr io.Reader) error { startTime := time.Now() - log.I.F("import: starting import operation") + log.I.F("================================================") + log.I.F("[HTTP API IMPORT] Starting import operation") + log.I.F("[HTTP API IMPORT] State: Initializing - preparing to receive file data") + log.I.F("================================================") // store to disk so we can return fast tmpPath := os.TempDir() + string(os.PathSeparator) + "orly" os.MkdirAll(tmpPath, 0700) tmp, err := os.CreateTemp(tmpPath, "") if chk.E(err) { + log.E.F("[HTTP API IMPORT] ERROR: Failed to create temp file: %v", err) + log.E.F("[HTTP API IMPORT] Action: Check disk space and temp directory permissions") return err } defer os.Remove(tmp.Name()) // Clean up temp file when done - log.I.F("import: buffering upload to %s", tmp.Name()) + log.I.F("[HTTP API IMPORT] State: Buffering - receiving file data from client") + log.I.F("[HTTP API IMPORT] Temp file: %s", tmp.Name()) bufferStart := time.Now() bytesBuffered, err := io.Copy(tmp, rr) if chk.E(err) { + log.E.F("[HTTP API IMPORT] ERROR: Failed to buffer file data: %v", err) + log.E.F("[HTTP API IMPORT] Action: Check network connection and file integrity") return err } bufferElapsed := time.Since(bufferStart) - log.I.F("import: buffered %.2f MB in %v (%.2f MB/sec)", - float64(bytesBuffered)/1024/1024, bufferElapsed.Round(time.Millisecond), - float64(bytesBuffered)/bufferElapsed.Seconds()/1024/1024) + fileSizeMB := float64(bytesBuffered) / 1024 / 1024 + bufferSpeedMBps := fileSizeMB / bufferElapsed.Seconds() + log.I.F("[HTTP API IMPORT] State: Buffering complete") + log.I.F("[HTTP API IMPORT] File size: %.2f MB | Buffer time: %v | Speed: %.2f MB/sec", + fileSizeMB, bufferElapsed.Round(time.Millisecond), bufferSpeedMBps) if _, err = tmp.Seek(0, 0); chk.E(err) { + log.E.F("[HTTP API IMPORT] ERROR: Failed to rewind temp file: %v", err) return err } - processErr := d.processJSONLEvents(ctx, tmp) + // Estimate number of events (rough estimate: ~500 bytes per event average) + estimatedEvents := int(float64(bytesBuffered) / 500) + log.I.F("[HTTP API IMPORT] State: Processing - reading and importing events") + log.I.F("[HTTP API IMPORT] Estimated events: ~%d (based on file size)", estimatedEvents) + log.I.F("[HTTP API IMPORT] Progress updates will appear every 5 seconds") + log.I.F("================================================") + processErr := d.processJSONLEvents(ctx, tmp, bytesBuffered) totalElapsed := time.Since(startTime) - log.I.F("import: total operation time: %v", totalElapsed.Round(time.Millisecond)) + log.I.F("================================================") + log.I.F("[HTTP API IMPORT] State: Completed") + log.I.F("[HTTP API IMPORT] Total time: %v", totalElapsed.Round(time.Millisecond)) + if processErr != nil { + log.E.F("[HTTP API IMPORT] Result: FAILED") + log.E.F("[HTTP API IMPORT] Error: %v", processErr) + log.E.F("[HTTP API IMPORT] Action: Check error details above, verify file format is valid JSONL") + log.I.F("================================================") + } else { + log.I.F("[HTTP API IMPORT] Result: SUCCESS") + log.I.F("[HTTP API IMPORT] All events have been imported and are now available on the relay") + log.I.F("================================================") + } return processErr } // ImportEventsFromStrings imports events from a slice of JSON strings with policy filtering -func (d *D) ImportEventsFromStrings(ctx context.Context, eventJSONs []string, policyManager interface{ CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) }) error { +func (d *D) ImportEventsFromStrings(ctx context.Context, eventJSONs []string, policyManager interface { + CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) +}) error { // Create a reader from the string slice reader := strings.NewReader(strings.Join(eventJSONs, "\n")) - return d.processJSONLEventsWithPolicy(ctx, reader, policyManager) + // Estimate total bytes (rough estimate) + totalBytes := int64(len(strings.Join(eventJSONs, "\n"))) + return d.processJSONLEventsWithPolicy(ctx, reader, policyManager, totalBytes) } // processJSONLEvents processes JSONL events from a reader -func (d *D) processJSONLEvents(ctx context.Context, rr io.Reader) error { - return d.processJSONLEventsWithPolicy(ctx, rr, nil) +func (d *D) processJSONLEvents(ctx context.Context, rr io.Reader, totalBytes int64) error { + return d.processJSONLEventsWithPolicy(ctx, rr, nil, totalBytes) } // processJSONLEventsWithPolicy processes JSONL events from a reader with optional policy filtering -func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, policyManager interface{ CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) }) error { +func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, policyManager interface { + CheckPolicy(action string, ev *event.E, pubkey []byte, remote string) (bool, error) +}, totalBytes int64) error { // Create a scanner to read the buffer line by line scan := bufio.NewScanner(rr) scanBuf := make([]byte, maxLen) @@ -80,17 +116,26 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli lastLogTime := startTime const logInterval = 5 * time.Second - var count, total, skipped, policyRejected, unmarshalErrors, saveErrors int + var count, bytesRead, skipped, policyRejected, unmarshalErrors, saveErrors int + var kind3Count, largeEventCount int // Track noisy event types + + // Track bytes read for progress percentage + bytesReadTotal := int64(0) + for scan.Scan() { select { case <-ctx.Done(): - log.I.F("import: context closed after %d events", count) + log.I.F("[HTTP API IMPORT] State: Cancelled") + log.I.F("[HTTP API IMPORT] Processed %d events before cancellation", count) return ctx.Err() default: } b := scan.Bytes() - total += len(b) + 1 + lineLen := len(b) + 1 // +1 for newline + bytesReadTotal += int64(lineLen) + bytesRead += lineLen + if len(b) < 1 { skipped++ continue @@ -101,28 +146,41 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli // return the pooled buffer on error ev.Free() unmarshalErrors++ - log.W.F("failed to unmarshal event: %v", err) + // Only log unmarshal errors if they're frequent (not spam) + if unmarshalErrors <= 10 || unmarshalErrors%100 == 0 { + log.W.F("[HTTP API IMPORT] Failed to unmarshal event #%d: %v", unmarshalErrors, err) + } continue } + // Track event kinds for summary (but don't log each one) + if ev.Kind == 3 { + kind3Count++ + } + if len(b) > 100000 { // Events larger than 100KB + largeEventCount++ + } + // Apply policy checking if policy manager is provided if policyManager != nil { // For sync imports, we treat events as coming from system/trusted source // Use nil pubkey and empty remote to indicate system-level import allowed, policyErr := policyManager.CheckPolicy("write", ev, nil, "") if policyErr != nil { - log.W.F("policy check failed for event %x: %v", ev.ID, policyErr) ev.Free() policyRejected++ + // Only log policy errors if they're frequent (not spam) + if policyRejected <= 10 || policyRejected%100 == 0 { + log.W.F("[HTTP API IMPORT] Policy check failed for event %x: %v", ev.ID, policyErr) + } continue } if !allowed { - log.D.F("policy rejected event %x during sync import", ev.ID) ev.Free() policyRejected++ + // Don't log individual policy rejections (too noisy) continue } - log.D.F("policy allowed event %x during sync import", ev.ID) } // Apply rate limiting before write operation if limiter is configured @@ -134,7 +192,10 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli // return the pooled buffer on error paths too ev.Free() saveErrors++ - log.W.F("failed to save event: %v", err) + // Only log save errors if they're frequent (not spam) + if saveErrors <= 10 || saveErrors%100 == 0 { + log.W.F("[HTTP API IMPORT] Failed to save event #%d: %v", saveErrors, err) + } continue } @@ -143,30 +204,99 @@ func (d *D) processJSONLEventsWithPolicy(ctx context.Context, rr io.Reader, poli b = nil count++ - // Progress logging every logInterval + // Progress logging every logInterval with detailed information if time.Since(lastLogTime) >= logInterval { elapsed := time.Since(startTime) eventsPerSec := float64(count) / elapsed.Seconds() - mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024 - log.I.F("import: progress %d events saved, %.2f MB read, %.0f events/sec, %.2f MB/sec", - count, float64(total)/1024/1024, eventsPerSec, mbPerSec) + mbRead := float64(bytesRead) / 1024 / 1024 + mbPerSec := mbRead / elapsed.Seconds() + + // Calculate progress percentage if we have total bytes + progressPercent := "" + eta := "" + if totalBytes > 0 { + percent := float64(bytesReadTotal) / float64(totalBytes) * 100 + if percent > 100 { + percent = 100 + } + progressPercent = fmt.Sprintf(" | Progress: %.1f%%", percent) + + // Estimate time remaining + if percent > 0 && percent < 100 && eventsPerSec > 0 { + remainingBytes := totalBytes - bytesReadTotal + remainingMB := float64(remainingBytes) / 1024 / 1024 + if mbPerSec > 0 { + remainingSeconds := remainingMB / mbPerSec + eta = fmt.Sprintf(" | ETA: ~%v", time.Duration(remainingSeconds)*time.Second) + } + } + } + + log.I.F("[HTTP API IMPORT] Progress: %d events saved | %.2f MB read | %.0f events/sec | %.2f MB/sec%s%s", + count, mbRead, eventsPerSec, mbPerSec, progressPercent, eta) + + // Show error summary if there are errors + if unmarshalErrors > 0 || saveErrors > 0 || policyRejected > 0 { + log.W.F("[HTTP API IMPORT] Errors so far: %d unmarshal, %d save, %d policy rejected", + unmarshalErrors, saveErrors, policyRejected) + } + lastLogTime = time.Now() debug.FreeOSMemory() } } - // Final summary + // Final summary with actionable information elapsed := time.Since(startTime) eventsPerSec := float64(count) / elapsed.Seconds() - mbPerSec := float64(total) / elapsed.Seconds() / 1024 / 1024 - log.I.F("import: completed - %d events saved, %.2f MB in %v (%.0f events/sec, %.2f MB/sec)", - count, float64(total)/1024/1024, elapsed.Round(time.Millisecond), eventsPerSec, mbPerSec) - if unmarshalErrors > 0 || saveErrors > 0 || policyRejected > 0 || skipped > 0 { - log.I.F("import: stats - %d unmarshal errors, %d save errors, %d policy rejected, %d skipped empty lines", - unmarshalErrors, saveErrors, policyRejected, skipped) + mbRead := float64(bytesRead) / 1024 / 1024 + mbPerSec := mbRead / elapsed.Seconds() + + log.I.F("[HTTP API IMPORT] ================================================") + log.I.F("[HTTP API IMPORT] Processing complete") + log.I.F("[HTTP API IMPORT] Events saved: %d | Data processed: %.2f MB | Time: %v", + count, mbRead, elapsed.Round(time.Millisecond)) + log.I.F("[HTTP API IMPORT] Performance: %.0f events/sec | %.2f MB/sec", eventsPerSec, mbPerSec) + + // Show statistics + if kind3Count > 0 || largeEventCount > 0 { + log.I.F("[HTTP API IMPORT] Event types: %d kind-3 (follow lists), %d large events (>100KB)", + kind3Count, largeEventCount) + } + + // Show error summary with actionable information + hasErrors := false + if unmarshalErrors > 0 { + log.W.F("[HTTP API IMPORT] WARNING: %d events failed to unmarshal", unmarshalErrors) + log.W.F("[HTTP API IMPORT] Action: Check if file format is valid JSONL (one JSON object per line)") + hasErrors = true + } + if saveErrors > 0 { + log.W.F("[HTTP API IMPORT] WARNING: %d events failed to save", saveErrors) + log.W.F("[HTTP API IMPORT] Action: Check database disk space and permissions") + hasErrors = true + } + if policyRejected > 0 { + log.W.F("[HTTP API IMPORT] WARNING: %d events rejected by policy", policyRejected) + log.W.F("[HTTP API IMPORT] Action: Review ACL/policy settings if this is unexpected") + hasErrors = true + } + if skipped > 0 { + log.I.F("[HTTP API IMPORT] Info: %d empty lines skipped (this is normal)", skipped) + } + + if !hasErrors { + log.I.F("[HTTP API IMPORT] Result: All events imported successfully") + log.I.F("[HTTP API IMPORT] Action: Events are now available on the relay") + } else { + log.W.F("[HTTP API IMPORT] Result: Import completed with errors (see warnings above)") + log.W.F("[HTTP API IMPORT] Action: Review error messages and take appropriate action") } + log.I.F("[HTTP API IMPORT] ================================================") if err := scan.Err(); err != nil { + log.E.F("[HTTP API IMPORT] ERROR: Scanner error: %v", err) + log.E.F("[HTTP API IMPORT] Action: Check file integrity and format") return err } diff --git a/run-orly-docker.sh b/run-orly-docker.sh index 5e3d5a3..db05c2f 100755 --- a/run-orly-docker.sh +++ b/run-orly-docker.sh @@ -1,11 +1,36 @@ #!/bin/bash # Run Orly relay using docker run (alternative to docker-compose) -# Optimized for large dataset imports (20GB+) +# Optimized for large dataset imports (20GB+) with memory-conscious settings +# +# Memory optimizations applied: +# - Reduced cache sizes (block: 256MB, index: 128MB) +# - Lower serial caches (pubkeys: 50k, event IDs: 250k) +# - Explicit rate limit target (1500MB) to prevent auto-detection issues +# - Fixed emergency thresholds (1.167/0.833 instead of 4.0/3.0) +# - Query result limit (256) to prevent unbounded memory usage +# - Reduced max connections (500 instead of 1000) set -e CONTAINER_NAME="orly-relay" -IMAGE="silberengel/next-orly:v0.58.5" +# Auto-detect version or use default +# Try multiple methods to detect version +VERSION="" +if command -v git &> /dev/null && [ -d .git ]; then + VERSION=$(git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//' || echo "") +fi +if [ -z "$VERSION" ] && [ -f pkg/version/version ]; then + VERSION=$(cat pkg/version/version 2>/dev/null | tr -d ' \n' | sed 's/^v//' || echo "") +fi +# Fallback to default if still empty +if [ -z "$VERSION" ]; then + VERSION="0.58.10" +fi +# Ensure version starts with 'v' +if [[ ! "$VERSION" =~ ^v ]]; then + VERSION="v${VERSION}" +fi +IMAGE="silberengel/next-orly:${VERSION}" # Data directory on host filesystem (change this to your desired path) # Using bind mount instead of volume for better performance with large datasets @@ -39,8 +64,9 @@ docker run -d \ -p 0.0.0.0:3334:3334 \ -p 0.0.0.0:7777:7777 \ -v "${DATA_DIR}:/data" \ - --memory=6144m \ + --memory=4096m \ --cpus="2.0" \ + --oom-kill-disable=false \ --health-cmd="curl -f http://localhost:7777/ || exit 1" \ --health-interval=10s \ --health-timeout=5s \ @@ -49,7 +75,8 @@ docker run -d \ -e ORLY_DATA_DIR=/data \ -e ORLY_LISTEN=0.0.0.0 \ -e ORLY_PORT=7777 \ - -e ORLY_LOG_LEVEL=Info \ + -e ORLY_LOG_LEVEL=Warn \ + -e NOSTR_PRIVATE_KEY="${NOSTR_PRIVATE_KEY:-}" \ -e ORLY_ADMINS=npub1m4ny6hjqzepn4rxknuq94c2gpqzr29ufkkw7ttcxyak7v43n6vvsajc2jl,npub1v30tsz9vw6ylpz63g0a702nj3xa26t3m7p5us8f2y2sd8v6cnsvq465zjx,npub12umrfdjgvdxt45g0y3ghwcyfagssjrv5qlm3t6pu2aa5vydwdmwq8q0z04,npub18cddpua960qjy3wmw7y9gmzr4h3ajlrwq3k9jnmqzlxke4qkg6gqeyaztw \ -e ORLY_OWNERS=npub1m4ny6hjqzepn4rxknuq94c2gpqzr29ufkkw7ttcxyak7v43n6vvsajc2jl,npub1v30tsz9vw6ylpz63g0a702nj3xa26t3m7p5us8f2y2sd8v6cnsvq465zjx,npub12umrfdjgvdxt45g0y3ghwcyfagssjrv5qlm3t6pu2aa5vydwdmwq8q0z04,npub18cddpua960qjy3wmw7y9gmzr4h3ajlrwq3k9jnmqzlxke4qkg6gqeyaztw \ -e ORLY_ACL_MODE=follows \ @@ -57,10 +84,10 @@ docker run -d \ -e ORLY_RELAY_URL=wss://orly-relay.imwald.eu \ -e ORLY_SPROCKET_ENABLED=false \ -e ORLY_DB_LOG_LEVEL=error \ - -e ORLY_DB_BLOCK_CACHE_MB=512 \ - -e ORLY_DB_INDEX_CACHE_MB=256 \ - -e ORLY_SERIAL_CACHE_PUBKEYS=100000 \ - -e ORLY_SERIAL_CACHE_EVENT_IDS=500000 \ + -e ORLY_DB_BLOCK_CACHE_MB=256 \ + -e ORLY_DB_INDEX_CACHE_MB=128 \ + -e ORLY_SERIAL_CACHE_PUBKEYS=50000 \ + -e ORLY_SERIAL_CACHE_EVENT_IDS=250000 \ -e ORLY_DB_ZSTD_LEVEL=9 \ -e ORLY_GC_ENABLED=true \ -e ORLY_GC_BATCH_SIZE=5000 \ @@ -68,13 +95,21 @@ docker run -d \ -e ORLY_BOOTSTRAP_RELAYS=wss://profiles.nostr1.com,wss://purplepag.es,wss://relay.nostr.band,wss://relay.damus.io \ -e ORLY_SUBSCRIPTION_ENABLED=false \ -e ORLY_MONTHLY_PRICE_SAT=0 \ - -e ORLY_MAX_CONNECTIONS=1000 \ + -e ORLY_MAX_CONNECTIONS=500 \ -e ORLY_MAX_EVENT_SIZE=65536 \ -e ORLY_MAX_SUBSCRIPTIONS=20 \ + -e ORLY_QUERY_RESULT_LIMIT=256 \ -e ORLY_WEB_DISABLE=false \ -e ORLY_WEB_DEV_PROXY_URL="" \ - -e ORLY_RATE_LIMIT_EMERGENCY_THRESHOLD=4.0 \ - -e ORLY_RATE_LIMIT_RECOVERY_THRESHOLD=3.0 \ + -e ORLY_RATE_LIMIT_ENABLED=true \ + -e ORLY_RATE_LIMIT_TARGET_MB=1500 \ + -e ORLY_RATE_LIMIT_WRITE_TARGET=0.70 \ + -e ORLY_RATE_LIMIT_WRITE_KP=1.0 \ + -e ORLY_RATE_LIMIT_WRITE_KI=0.2 \ + -e ORLY_RATE_LIMIT_MAX_WRITE_MS=2000 \ + -e ORLY_RATE_LIMIT_EMERGENCY_THRESHOLD=1.167 \ + -e ORLY_RATE_LIMIT_RECOVERY_THRESHOLD=0.833 \ + -e ORLY_RATE_LIMIT_EMERGENCY_MAX_MS=5000 \ ${IMAGE} echo "" diff --git a/scripts/app/web/dist/index.html b/scripts/app/web/dist/index.html new file mode 100644 index 0000000..9cbf916 --- /dev/null +++ b/scripts/app/web/dist/index.html @@ -0,0 +1 @@ + diff --git a/scripts/cleanup-stuck-imports.sh b/scripts/cleanup-stuck-imports.sh new file mode 100755 index 0000000..eaa41d5 --- /dev/null +++ b/scripts/cleanup-stuck-imports.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Cleanup stuck import containers on remote server +# Usage: ./scripts/cleanup-stuck-imports.sh [remote_host] [remote_user] + +REMOTE_HOST="${REMOTE_HOST:-${1:-217.154.126.125}}" +REMOTE_USER="${REMOTE_USER:-${2:-root}}" + +echo "Cleaning up stuck import containers on ${REMOTE_USER}@${REMOTE_HOST}..." + +# Find and stop all containers running orly db import +ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --filter 'ancestor=silberengel/next-orly:v0.58.10' --format '{{.ID}} {{.Command}}' | grep -E 'db import|sh -c.*orly db' | awk '{print \$1}'" | while read container_id; do + if [ -n "$container_id" ]; then + echo "Stopping container: $container_id" + ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker stop $container_id 2>/dev/null && docker rm $container_id 2>/dev/null" || true + fi +done + +echo "Cleanup complete. Remaining orly containers:" +ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps -a | grep orly || echo 'No orly containers found'" diff --git a/scripts/create-test-event.go b/scripts/create-test-event.go new file mode 100644 index 0000000..00a4457 --- /dev/null +++ b/scripts/create-test-event.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "os" + "time" + + "git.mleku.dev/mleku/nostr/encoders/bech32encoding" + "git.mleku.dev/mleku/nostr/encoders/event" + "git.mleku.dev/mleku/nostr/encoders/kind" + "git.mleku.dev/mleku/nostr/encoders/tag" + "git.mleku.dev/mleku/nostr/interfaces/signer/p8k" +) + +func main() { + // Get NOSTR_PRIVATE_KEY from environment + nsec := os.Getenv("NOSTR_PRIVATE_KEY") + if nsec == "" { + fmt.Fprintf(os.Stderr, "ERROR: NOSTR_PRIVATE_KEY environment variable not set\n") + os.Exit(1) + } + + // Decode nsec to get private key bytes + secretBytes, err := bech32encoding.NsecToBytes([]byte(nsec)) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to decode nsec: %v\n", err) + os.Exit(1) + } + + // Create signer and initialize with private key + signer, err := p8k.New() + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to create signer: %v\n", err) + os.Exit(1) + } + if err = signer.InitSec(secretBytes); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to initialize signer: %v\n", err) + os.Exit(1) + } + + // Create a test event (kind 1 - text note) + ev := event.New() + ev.CreatedAt = time.Now().Unix() + ev.Kind = kind.TextNote.K + ev.Content = []byte("Pre-flight test event for import verification") + ev.Tags = tag.NewS() + + // Sign the event + if err := ev.Sign(signer); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to sign event: %v\n", err) + os.Exit(1) + } + + // Serialize to JSON + eventJSON := ev.Serialize() + + // Output the JSON event + fmt.Println(string(eventJSON)) +} diff --git a/scripts/import-exports-remote.sh b/scripts/import-exports-remote.sh new file mode 100755 index 0000000..514f7ac --- /dev/null +++ b/scripts/import-exports-remote.sh @@ -0,0 +1,754 @@ +#!/bin/bash + +# Import JSONL files to remote Orly relay +# This script copies files from local exports directory to remote server and imports them +# Usage: ./scripts/import-exports-remote.sh [remote_host] [remote_user] + +set -e + +# Configuration +REMOTE_HOST="${REMOTE_HOST:-${1:-217.154.126.125}}" +REMOTE_USER="${REMOTE_USER:-${2:-root}}" +REMOTE_TEMP_DIR="${REMOTE_TEMP_DIR:-/root/tmp/orly}" +DOCKER_CONTAINER="${DOCKER_CONTAINER:-orly-relay}" +# NOSTR_PRIVATE_KEY for localhost authentication (reads from environment, e.g., .bashrc) +NOSTR_PRIVATE_KEY="${NOSTR_PRIVATE_KEY:-}" + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' + +# Get script directory and calculate exports path +# Script is in scripts/, exports are in ../scripts/exports relative to script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPORTS_DIR="$(cd "$SCRIPT_DIR/../../scripts/exports" 2>/dev/null && pwd || echo "")" + +# IMPORTANT: This script NEVER deletes source files from EXPORTS_DIR +# It only copies files to remote server and moves them to a "done" folder after successful import + +# Helper function to query an event ID from the relay +# Returns 0 if event found, 1 if not found +query_event_on_relay() { + local event_id="$1" + local relay_ws_http="ws://${REMOTE_HOST}:7777/" + local relay_ws_https="wss://${REMOTE_HOST}:7777/" + local event_found=0 + + for relay_ws in "$relay_ws_http" "$relay_ws_https"; do + if command -v websocat >/dev/null 2>&1; then + local query_msg="[\"REQ\",\"verify-query-$(date +%s)\",{\"ids\":[\"${event_id}\"]}]" + local query_result=$(timeout 10 bash -c "echo '$query_msg' | websocat '$relay_ws' 2>&1" | head -20) + + if echo "$query_result" | grep -q "\"id\":\"${event_id}\""; then + return 0 + fi + elif command -v python3 >/dev/null 2>&1; then + python3 < 0: + if data[0] == "EVENT" and len(data) > 2: + event_data = data[2] + if isinstance(event_data, dict) and event_data.get("id") == event_id: + found_event = True + break + elif data[0] == "EOSE": + # End of stored events - stop waiting + break + except asyncio.TimeoutError: + pass + except Exception: + pass + + return found_event + except Exception: + return False + +relay_url = "${relay_ws}" +event_id = "${event_id}" +result = asyncio.run(query_event(relay_url, event_id)) +if result: + sys.exit(0) +else: + sys.exit(1) +PYEOF + if [ $? -eq 0 ]; then + return 0 + fi + fi + done + + return 1 +} + +# Check if exports directory exists +if [ -z "$EXPORTS_DIR" ] || [ ! -d "$EXPORTS_DIR" ]; then + echo -e "${RED}Error: Exports directory not found at: $SCRIPT_DIR/../scripts/exports${NC}" + echo "Expected path: $(cd "$SCRIPT_DIR/.." && pwd)/scripts/exports" + exit 1 +fi + +# Find all JSONL files +jsonl_files=("$EXPORTS_DIR"/*.jsonl) + +# Check if any JSONL files exist +if [ ! -e "${jsonl_files[0]}" ]; then + echo -e "${YELLOW}No JSONL files found in: $EXPORTS_DIR${NC}" + exit 0 +fi + +# Count files +file_count=0 +for file in "${jsonl_files[@]}"; do + if [ -f "$file" ]; then + file_count=$((file_count + 1)) + fi +done + +if [ "$file_count" -eq 0 ]; then + echo -e "${YELLOW}No JSONL files found in: $EXPORTS_DIR${NC}" + exit 0 +fi + +echo -e "${BLUE}=== Import Exports to Remote Server ===${NC}" +echo "" +echo "Exports directory: $EXPORTS_DIR" +echo "Remote server: ${REMOTE_USER}@${REMOTE_HOST}" +echo "Remote temp directory: ${REMOTE_TEMP_DIR}" +echo "Docker container: ${DOCKER_CONTAINER}" +echo "Files found: $file_count" +echo "" + +# Check if SSH is available +if ! command -v ssh >/dev/null 2>&1; then + echo -e "${RED}Error: ssh command not found. Please install OpenSSH client.${NC}" + exit 1 +fi + +# Check if SCP is available +if ! command -v scp >/dev/null 2>&1; then + echo -e "${RED}Error: scp command not found. Please install OpenSSH client.${NC}" + exit 1 +fi + +# Test SSH connection +echo -e "${CYAN}Testing SSH connection to ${REMOTE_USER}@${REMOTE_HOST}...${NC}" +if ! ssh -o ConnectTimeout=10 -o BatchMode=yes "${REMOTE_USER}@${REMOTE_HOST}" "echo 'Connection successful'" 2>/dev/null; then + echo -e "${YELLOW}Warning: SSH connection test failed. You may be prompted for password/key.${NC}" + echo "Continuing anyway..." +fi +echo "" + +# Check if Docker container exists on remote +echo -e "${CYAN}Checking Docker container on remote server...${NC}" +if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --format '{{.Names}}' | grep -q '^${DOCKER_CONTAINER}$'" 2>/dev/null; then + echo -e "${RED}Error: Docker container '${DOCKER_CONTAINER}' not found on remote server.${NC}" + echo "Available containers:" + ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --format '{{.Names}}'" 2>/dev/null || true + exit 1 +fi +echo -e "${GREEN}✓ Docker container '${DOCKER_CONTAINER}' found${NC}" +echo "" + +# Check if NOSTR_PRIVATE_KEY is available (local or in container) +echo -e "${CYAN}Checking for NOSTR_PRIVATE_KEY...${NC}" +container_has_key=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} sh -c 'echo \$NOSTR_PRIVATE_KEY'" 2>/dev/null | grep -v '^$' || echo "") + +if [ -n "$container_has_key" ]; then + echo -e "${GREEN}✓ NOSTR_PRIVATE_KEY: Found in container (will use for localhost authentication)${NC}" +elif [ -n "$NOSTR_PRIVATE_KEY" ]; then + echo -e "${GREEN}✓ NOSTR_PRIVATE_KEY: Found in local environment${NC}" + echo -e "${CYAN} Will pass it to container for localhost authentication${NC}" +else + echo -e "${YELLOW}⚠ NOSTR_PRIVATE_KEY: Not found${NC}" + echo -e "${YELLOW} Set it locally: export NOSTR_PRIVATE_KEY=nsec1your_key_here${NC}" + echo -e "${YELLOW} Or add to container environment when running docker${NC}" +fi +echo "" + +# Create temp directory on remote (needed for pre-flight test) +echo -e "${CYAN}Creating temporary directory on remote server...${NC}" +ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p '${REMOTE_TEMP_DIR}'" || { + echo -e "${RED}Error: Failed to create temp directory on remote server${NC}" + exit 1 +} +echo -e "${GREEN}✓ Temp directory created${NC}" +echo "" + +# Pre-flight test: Send a test event and verify it appears on the relay +# This runs BEFORE copying files to catch configuration issues early +echo -e "${CYAN}=== Pre-flight Test: Verifying Import Works ===${NC}" +echo " Creating and sending a test event..." +echo " (This verifies configuration before copying large files)" + +# Determine which NOSTR_PRIVATE_KEY to use +TEST_NOSTR_KEY="" +if [ -n "$container_has_key" ]; then + TEST_NOSTR_KEY="$container_has_key" + echo " Using NOSTR_PRIVATE_KEY from container for test event" +elif [ -n "$NOSTR_PRIVATE_KEY" ]; then + TEST_NOSTR_KEY="$NOSTR_PRIVATE_KEY" + echo " Using NOSTR_PRIVATE_KEY from local environment for test event" +else + echo -e "${RED}✗ NOSTR_PRIVATE_KEY not found - cannot create signed test event${NC}" + echo -e "${RED}✗ Pre-flight test FAILED - Need NOSTR_PRIVATE_KEY for write permissions${NC}" + exit 1 +fi + +# Create a test event using Go +TEST_EVENT_FILE=$(mktemp /tmp/orly-test-event-XXXXXX.jsonl) +TEST_EVENT_ID="" + +# Get the script directory to find the Go program +GO_PROGRAM="${SCRIPT_DIR}/create-test-event.go" + +# Compile and run the Go program +if command -v go >/dev/null 2>&1; then + echo " Creating signed test event with Go..." + + # Compile the Go program to a temporary binary + TEMP_BINARY=$(mktemp /tmp/orly-test-event-XXXXXX) + TEMP_ERR=$(mktemp /tmp/orly-test-event-err-XXXXXX) + + if ! go build -o "$TEMP_BINARY" "$GO_PROGRAM" 2>"$TEMP_ERR"; then + COMPILE_ERR=$(cat "$TEMP_ERR" 2>/dev/null || echo "") + echo -e "${RED}✗ Failed to compile test event generator${NC}" + if [ -n "$COMPILE_ERR" ]; then + echo " Compile error: $COMPILE_ERR" + fi + rm -f "$TEST_EVENT_FILE" "$TEMP_BINARY" "$TEMP_ERR" + exit 1 + fi + rm -f "$TEMP_ERR" + + # Run it with NOSTR_PRIVATE_KEY + TEST_EVENT_JSON=$(NOSTR_PRIVATE_KEY="$TEST_NOSTR_KEY" "$TEMP_BINARY" 2>"$TEMP_ERR") + EXIT_CODE=$? + ERROR_OUTPUT=$(cat "$TEMP_ERR" 2>/dev/null || echo "") + + # Clean up binary and error file + rm -f "$TEMP_BINARY" "$TEMP_ERR" + + if [ $EXIT_CODE -ne 0 ] || [ -z "$TEST_EVENT_JSON" ]; then + echo -e "${RED}✗ Failed to create signed test event${NC}" + if [ -n "$ERROR_OUTPUT" ]; then + echo " Error: $ERROR_OUTPUT" + fi + rm -f "$TEST_EVENT_FILE" + exit 1 + fi + + # Save the event JSON + echo "$TEST_EVENT_JSON" > "$TEST_EVENT_FILE" + + # Extract event ID + if command -v jq >/dev/null 2>&1; then + TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | jq -r '.id' 2>/dev/null || echo "") + elif command -v python3 >/dev/null 2>&1; then + TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | python3 -c "import sys, json; print(json.load(sys.stdin).get('id', ''))" 2>/dev/null || echo "") + else + # Fallback: extract ID using grep/sed + TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | grep -o '"id":"[^"]*"' | sed 's/"id":"\([^"]*\)"/\1/' | head -1) + fi +else + echo -e "${RED}✗ Go compiler not found - cannot create signed test event${NC}" + echo -e "${YELLOW} Install Go: https://golang.org/dl/${NC}" + exit 1 +fi + +if [ -z "$TEST_EVENT_ID" ] || [ ! -s "$TEST_EVENT_FILE" ]; then + echo -e "${RED}✗ Failed to create test event or extract event ID${NC}" + rm -f "$TEST_EVENT_FILE" + exit 1 +fi + +echo " Test event ID: ${TEST_EVENT_ID:0:16}..." +echo " Uploading test event to container..." + +# Copy test event to remote server first, then into container +REMOTE_TEMP_EVENT="/tmp/orly-test-event-$(date +%s).jsonl" +if ! scp "$TEST_EVENT_FILE" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_TEMP_EVENT}" 2>&1; then + echo -e "${RED}✗ Failed to copy test event to remote server${NC}" + rm -f "$TEST_EVENT_FILE" + exit 1 +fi + +# Now copy from remote server into container +if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker cp ${REMOTE_TEMP_EVENT} ${DOCKER_CONTAINER}:/tmp/test_event.jsonl" 2>&1; then + echo -e "${RED}✗ Failed to copy test event to container${NC}" + ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_TEMP_EVENT}" 2>/dev/null || true + rm -f "$TEST_EVENT_FILE" + exit 1 +fi + +# Fix ownership and permissions so the orly user can read it +ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u root ${DOCKER_CONTAINER} chown orly:orly /tmp/test_event.jsonl && docker exec -u root ${DOCKER_CONTAINER} chmod 644 /tmp/test_event.jsonl" 2>/dev/null || true + +# Clean up remote temp file +ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_TEMP_EVENT}" 2>/dev/null || true + +# Send test event via import API +echo " Sending test event via import API..." +REMOTE_RESPONSE_FILE="/tmp/orly-test-response-$(date +%s).txt" + +if [ -n "$container_has_key" ]; then + ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time 25 --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/test_event.jsonl\" \"http://127.0.0.1:7777/api/import?async=true\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1 + CURL_EXIT=$? +elif [ -n "$NOSTR_PRIVATE_KEY" ]; then + escaped_key=$(echo "$NOSTR_PRIVATE_KEY" | sed "s/'/'\\\\''/g") + ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} sh -c \"export NOSTR_PRIVATE_KEY=\\\"$escaped_key\\\" && curl --max-time 25 --connect-timeout 5 -s -w \\\"\\\\n%{http_code}\\\" -X POST -F \\\"file=@/tmp/test_event.jsonl\\\" \\\"http://127.0.0.1:7777/api/import?async=true\\\"\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1 + CURL_EXIT=$? +else + ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time 25 --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/test_event.jsonl\" \"http://127.0.0.1:7777/api/import?async=true\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1 + CURL_EXIT=$? +fi + +# Retrieve the response file +test_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_RESPONSE_FILE} 2>/dev/null" 2>/dev/null || echo "") +ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_RESPONSE_FILE}" 2>/dev/null || true + +# Check if curl timed out or failed +if [ $CURL_EXIT -eq 124 ]; then + echo -e "${RED}✗ Test upload timed out after 30 seconds${NC}" + rm -f "$TEST_EVENT_FILE" + ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true + exit 1 +elif [ $CURL_EXIT -ne 0 ]; then + echo -e "${RED}✗ Test upload failed (exit code: $CURL_EXIT)${NC}" + rm -f "$TEST_EVENT_FILE" + ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true + exit 1 +fi + +test_http_code=$(echo "$test_response" | tail -n1 | tr -d '[:space:]') +test_body=$(echo "$test_response" | sed '$d') + +if [ "$test_http_code" -ne 200 ] && [ "$test_http_code" -ne 202 ]; then + echo -e "${RED}✗ Test upload failed (HTTP $test_http_code)${NC}" + rm -f "$TEST_EVENT_FILE" + ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true + exit 1 +fi + +echo -e "${GREEN}✓ Test event uploaded successfully (HTTP $test_http_code)${NC}" +echo " Waiting 5 seconds for event to be processed..." +sleep 5 + +echo " Querying relay for test event..." +RELAY_WS_HTTP="ws://${REMOTE_HOST}:7777/" +RELAY_WS_HTTPS="wss://${REMOTE_HOST}:7777/" +RELAY_WS="" +EVENT_FOUND=0 + +# Try to query using websocat or Python +for RELAY_WS in "$RELAY_WS_HTTP" "$RELAY_WS_HTTPS"; do + echo " Trying ${RELAY_WS}..." + + if command -v websocat >/dev/null 2>&1; then + QUERY_MSG="[\"REQ\",\"test-query-$(date +%s)\",{\"ids\":[\"${TEST_EVENT_ID}\"]}]" + QUERY_RESULT=$(timeout 10 bash -c "echo '$QUERY_MSG' | websocat '$RELAY_WS' 2>&1" | head -20) + + if echo "$QUERY_RESULT" | grep -q "\"id\":\"${TEST_EVENT_ID}\""; then + EVENT_FOUND=1 + break + fi + elif command -v python3 >/dev/null 2>&1; then + echo " Using Python websockets to query..." + QUERY_RESULT=$(python3 < 0: + if data[0] == "EVENT" and len(data) > 2: + event_data = data[2] + if isinstance(event_data, dict) and event_data.get("id") == event_id: + found_event = True + break + elif data[0] == "EOSE": + # End of stored events - stop waiting + break + except asyncio.TimeoutError: + pass + except Exception: + pass + + return found_event + except Exception: + return False + +relay_url = "${RELAY_WS}" +event_id = "${TEST_EVENT_ID}" +result = asyncio.run(query_event(relay_url, event_id)) +if result: + sys.exit(0) +else: + sys.exit(1) +PYEOF + ) + QUERY_EXIT=$? + + if [ $QUERY_EXIT -eq 0 ]; then + EVENT_FOUND=1 + break + fi + else + echo -e "${YELLOW} Warning: Neither websocat nor Python websockets available${NC}" + break + fi +done + +# Clean up test files +rm -f "$TEST_EVENT_FILE" +ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true + +if [ "$EVENT_FOUND" -eq 1 ]; then + echo -e "${GREEN}✓ Test event found on relay${NC}" + echo -e "${GREEN}✓ Pre-flight test PASSED - Import is working${NC}" +else + echo -e "${RED}✗ Test event NOT found on relay${NC}" + echo -e "${RED}✗ Pre-flight test FAILED - Import is not working${NC}" + echo -e "${YELLOW} Fix configuration issues before proceeding${NC}" + exit 1 +fi + +echo "" +echo -e "${GREEN}✓ All pre-flight checks passed. Proceeding with file copy and import...${NC}" +echo "" + +# Process files one at a time: copy, extract event ID, import, verify, move to done +echo -e "${BLUE}=== Processing Files One at a Time ===${NC}" +echo -e "${CYAN}Method: Using HTTP API import (synchronous, no container restart needed)${NC}" +echo "" + +success_count=0 +fail_count=0 +current=0 +DONE_DIR="${REMOTE_TEMP_DIR}/done" + +# Ensure temp and done directories exist on remote +ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p '${REMOTE_TEMP_DIR}' '${DONE_DIR}'" 2>/dev/null || true + +# Process each local file one at a time +for local_file in "${jsonl_files[@]}"; do + if [ ! -f "$local_file" ]; then + continue + fi + + current=$((current + 1)) + filename=$(basename "$local_file") + file_size=$(du -h "$local_file" | cut -f1) + remote_file="${REMOTE_TEMP_DIR}/${filename}" + + echo "" + echo -e "${BLUE}=== [$current/$file_count] Processing: $filename (${file_size}) ===${NC}" + echo "" + + # Step 1: Copy file to remote server + echo -e "${CYAN}[1/5] Copying file to remote server...${NC}" + if ! scp "$local_file" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_TEMP_DIR}/" 2>&1; then + echo -e "${RED}✗ Failed to copy: $filename${NC}" + fail_count=$((fail_count + 1)) + echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" + continue + fi + echo -e "${GREEN}✓ Copied: $filename${NC}" + echo "" + + # Step 2: Extract event ID from the remote file + echo -e "${CYAN}[2/5] Extracting event ID from file...${NC}" + file_event_id="" + + # Extract first event ID from the file (look for "id":"..." pattern) + # Try multiple patterns to handle different JSON formats + file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -oE '\"id\"[[:space:]]*:[[:space:]]*\"[a-f0-9]{64}\"' | head -1 | sed -E 's/\"id\"[[:space:]]*:[[:space:]]*\"([^\"]+)\"/\1/'" || echo "") + + # If that didn't work, try simpler pattern + if [ -z "$file_event_id" ]; then + file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -o '\"id\":\"[a-f0-9]\{64\}\"' | head -1 | sed 's/\"id\":\"\([^\"]*\)\"/\1/'" || echo "") + fi + + # If still no ID, try to find any 64-char hex string that looks like an event ID + if [ -z "$file_event_id" ]; then + file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -oE '[a-f0-9]{64}' | head -1" || echo "") + fi + + if [ -n "$file_event_id" ]; then + echo -e "${GREEN}✓ Extracted event ID: ${file_event_id:0:16}...${NC}" + else + echo -e "${YELLOW}⚠ Could not extract event ID (will skip verification)${NC}" + fi + echo "" + + # Step 3: Import the file via HTTP API + echo -e "${CYAN}[3/5] Importing file via HTTP API...${NC}" + + # Check file size first + file_size_bytes=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "stat -c%s '${remote_file}' 2>/dev/null" || echo "0") + file_size_mb=$(echo "scale=2; $file_size_bytes / 1024 / 1024" | bc 2>/dev/null || echo "0") + echo " File size: ${file_size_mb} MB" + + import_success=false + + # Copy file into container for HTTP API import + echo " Copying file into container..." + if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker cp '${remote_file}' ${DOCKER_CONTAINER}:/tmp/" 2>&1; then + echo -e "${RED}✗ Failed to copy file into container${NC}" + fail_count=$((fail_count + 1)) + echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" + continue + fi + + # Fix ownership and permissions + ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u root ${DOCKER_CONTAINER} chown orly:orly /tmp/${filename} && docker exec -u root ${DOCKER_CONTAINER} chmod 644 /tmp/${filename}" 2>/dev/null || true + + # Calculate timeout based on file size + timeout_seconds=$((file_size_bytes / 1024 / 1024 * 30)) # ~30 seconds per MB + if [ $timeout_seconds -lt 300 ]; then + timeout_seconds=300 # Minimum 5 minutes + elif [ $timeout_seconds -gt 7200 ]; then + timeout_seconds=7200 # Maximum 2 hours + fi + echo " Using timeout: ${timeout_seconds}s for ${file_size_mb} MB file" + echo "" + + # Start streaming container logs in background to show import progress + # Use prominent log prefix so it's easy to find in logs + LOG_PREFIX="[IMPORT ${filename}]" + echo -e "${CYAN} Starting import via HTTP API (synchronous mode)...${NC}" + echo -e "${CYAN} Streaming container logs with prefix: ${LOG_PREFIX}${NC}" + echo "" + + # Start log streaming in background + LOG_TAIL_PID="" + (ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker logs -f --tail 0 ${DOCKER_CONTAINER} 2>&1" | while IFS= read -r line; do + # Show all lines with [HTTP API IMPORT] prefix (our new prominent logging) + # Also show other import-related lines for context + if echo "$line" | grep -qiE "\[HTTP API IMPORT\]|import.*progress|import.*complete|import.*failed|import.*error"; then + echo -e "${CYAN} ${LOG_PREFIX} $line${NC}" >&2 + fi + done) & + LOG_TAIL_PID=$! + sleep 1 # Give log stream a moment to start + + # Get NOSTR_PRIVATE_KEY for authentication + container_has_key=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} sh -c 'echo \$NOSTR_PRIVATE_KEY'" 2>/dev/null | grep -v '^$' || echo "") + + REMOTE_IMPORT_RESPONSE="/tmp/orly-import-response-$(date +%s).txt" + + # Show start time + UPLOAD_START=$(date +%s) + echo " [$(date +%H:%M:%S)] Starting HTTP API import..." + echo -e "${CYAN} ${LOG_PREFIX} Import started at $(date)${NC}" + + # Run import via HTTP API (synchronous - waits for completion) + if [ -n "$container_has_key" ]; then + echo " Using NOSTR_PRIVATE_KEY from container (localhost authentication)" + ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/${filename}\" \"http://127.0.0.1:7777/api/import\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1 + CURL_EXIT=$? + elif [ -n "$NOSTR_PRIVATE_KEY" ]; then + echo " Passing NOSTR_PRIVATE_KEY to container for localhost authentication" + escaped_key=$(echo "$NOSTR_PRIVATE_KEY" | sed "s/'/'\\\\''/g") + ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} sh -c \"export NOSTR_PRIVATE_KEY=\\\"$escaped_key\\\" && curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \\\"\\\\n%{http_code}\\\" -X POST -F \\\"file=@/tmp/${filename}\\\" \\\"http://127.0.0.1:7777/api/import\\\"\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1 + CURL_EXIT=$? + else + echo -e "${YELLOW} Warning: No NOSTR_PRIVATE_KEY available${NC}" + echo " Trying without authentication (may fail if ACL requires auth)..." + ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/${filename}\" \"http://127.0.0.1:7777/api/import\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1 + CURL_EXIT=$? + fi + + UPLOAD_END=$(date +%s) + UPLOAD_DURATION=$((UPLOAD_END - UPLOAD_START)) + echo " [$(date +%H:%M:%S)] HTTP request completed in ${UPLOAD_DURATION}s" + + # Stop log streaming + if [ -n "$LOG_TAIL_PID" ] && kill -0 "$LOG_TAIL_PID" 2>/dev/null; then + kill "$LOG_TAIL_PID" 2>/dev/null || true + wait "$LOG_TAIL_PID" 2>/dev/null || true + fi + + # Get response + import_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_IMPORT_RESPONSE} 2>/dev/null" 2>/dev/null || echo "") + ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_IMPORT_RESPONSE}" 2>/dev/null || true + + # Clean up file from container + ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f '/tmp/${filename}'" 2>/dev/null || true + + # Check if curl timed out + if [ $CURL_EXIT -eq 124 ]; then + echo -e "${RED}✗ Import timed out after ${timeout_seconds} seconds${NC}" + echo -e "${CYAN} ${LOG_PREFIX} Import timed out${NC}" + fail_count=$((fail_count + 1)) + echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" + continue + elif [ $CURL_EXIT -ne 0 ]; then + echo -e "${RED}✗ Import failed (exit code: $CURL_EXIT)${NC}" + echo -e "${CYAN} ${LOG_PREFIX} Import failed with exit code $CURL_EXIT${NC}" + fail_count=$((fail_count + 1)) + echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" + continue + fi + + # Extract HTTP status code and body + import_http_code=$(echo "$import_response" | tail -n1 | tr -d '[:space:]') + import_body=$(echo "$import_response" | sed '$d') + + # Check if import was successful + if [ -z "$import_http_code" ] || ! [[ "$import_http_code" =~ ^[0-9]+$ ]]; then + fail_count=$((fail_count + 1)) + echo -e "${RED}✗ Failed to import: $filename (invalid HTTP response)${NC}" + echo -e "${CYAN} ${LOG_PREFIX} Invalid HTTP response${NC}" + elif [ "$import_http_code" -eq 200 ]; then + if echo "$import_body" | grep -qi '"success":\s*true\|"message".*completed'; then + echo -e "${GREEN}✓ Imported: $filename (HTTP $import_http_code)${NC}" + echo -e "${CYAN} ${LOG_PREFIX} Import completed successfully${NC}" + import_success=true + success_count=$((success_count + 1)) + else + fail_count=$((fail_count + 1)) + echo -e "${RED}✗ Failed to import: $filename (HTTP $import_http_code, but no success in response)${NC}" + echo -e "${CYAN} ${LOG_PREFIX} Import response indicates failure${NC}" + fi + else + fail_count=$((fail_count + 1)) + echo -e "${RED}✗ Failed to import: $filename (HTTP $import_http_code)${NC}" + echo -e "${CYAN} ${LOG_PREFIX} Import failed with HTTP $import_http_code${NC}" + if [ -n "$import_body" ]; then + echo " Response: $import_body" + fi + fi + + # Step 4: Verify import + if [ "$import_success" = true ]; then + echo "" + echo -e "${CYAN}[4/5] Verifying import...${NC}" + if [ -n "$file_event_id" ]; then + echo " Verifying imported event exists on relay..." + sleep 3 # Give relay a moment to process the imported events + if query_event_on_relay "$file_event_id"; then + echo -e "${GREEN}✓ Verified: Event ${file_event_id:0:16}... from ${filename} found on relay${NC}" + else + echo -e "${YELLOW}⚠ Warning: Event ${file_event_id:0:16}... from ${filename} not yet found on relay (may need more time)${NC}" + fi + else + echo -e "${YELLOW}⚠ Skipping verification (no event ID extracted)${NC}" + fi + else + echo "" + echo -e "${CYAN}[4/5] Skipping verification (import failed)${NC}" + fi + echo "" + + # Step 5: Move to done folder (only if import was successful) + if [ "$import_success" = true ]; then + echo -e "${CYAN}[5/5] Moving file to done folder...${NC}" + if ssh "${REMOTE_USER}@${REMOTE_HOST}" "test -f '${remote_file}'" 2>/dev/null; then + if ssh "${REMOTE_USER}@${REMOTE_HOST}" "mv '${remote_file}' '${DONE_DIR}/${filename}'" 2>/dev/null; then + echo -e "${GREEN}✓ Moved: ${filename} to ${DONE_DIR}${NC}" + else + echo -e "${YELLOW}⚠ Warning: Could not move ${filename} to done folder${NC}" + fi + else + echo -e "${YELLOW}⚠ Warning: File ${filename} not found on remote (may have been moved already)${NC}" + fi + else + echo -e "${CYAN}[5/5] Skipping move to done (import failed - file remains in ${REMOTE_TEMP_DIR})${NC}" + fi + echo "" +done + +# Summary +echo "" +echo -e "${BLUE}=== Import Summary ===${NC}" +echo -e "${GREEN}Files succeeded: $success_count${NC}" +if [ "$fail_count" -gt 0 ]; then + echo -e "${RED}Files failed: $fail_count${NC}" +fi +echo "" + +if [ "$fail_count" -eq 0 ]; then + echo -e "${GREEN}All files imported successfully!${NC}" + echo -e "${GREEN}All successful files have been moved to ${DONE_DIR}${NC}" + exit 0 +else + echo -e "${YELLOW}Some imports failed. Check the errors above.${NC}" + echo -e "${YELLOW}Failed files remain in ${REMOTE_TEMP_DIR} for retry${NC}" + echo -e "${GREEN}Successful files have been moved to ${DONE_DIR}${NC}" + exit 1 +fi