Update event import process and improve user feedback

Simplified event import to run synchronously, ensuring proper resource handling and accurate feedback. Enhanced frontend with real-time import status messages and error handling. Adjusted migrations to handle events individually, improving reliability and granular progress tracking.

.claude/settings.local.json

@@ -12,7 +12,8 @@
       "Write:*",
       "Bash(go build:*)",
       "Bash(go test:*)",
-      "Bash(./scripts/test.sh:*)"
+      "Bash(./scripts/test.sh:*)",
+      "Bash(./scripts/update-embedded-web.sh:*)"
     ],
     "deny": [],
     "ask": []

app/web/src/App.svelte

@@ -53,6 +53,7 @@
     let searchTabs = [];
     let allEvents = [];
     let selectedFile = null;
+    let importMessage = ""; // Message shown after import completes
     let expandedEvents = new Set();
     let isLoadingEvents = false;
     let hasMoreEvents = true;
@@ -2356,22 +2357,28 @@
 
     // Import functionality
     function handleFileSelect(event) {
-        selectedFile = event.target.files[0];
+        // event.detail contains the original DOM event from the child component
+        selectedFile = event.detail.target.files[0];
     }
 
     async function importEvents() {
         // Skip login/permission check when ACL is "none" (open relay mode)
         if (aclMode !== "none" && (!isLoggedIn || (userRole !== "admin" && userRole !== "owner"))) {
-            alert("Admin or owner permission required");
+            importMessage = "Admin or owner permission required";
+            setTimeout(() => { importMessage = ""; }, 5000);
             return;
         }
 
         if (!selectedFile) {
-            alert("Please select a file");
+            importMessage = "Please select a file";
+            setTimeout(() => { importMessage = ""; }, 5000);
             return;
         }
 
         try {
+            // Show uploading message
+            importMessage = "Uploading...";
+
             // Build headers - only include auth when ACL is not "none"
             const headers = {};
             if (aclMode !== "none" && isLoggedIn) {
@@ -2396,12 +2403,15 @@
             }
 
             const result = await response.json();
-            alert("Import started successfully");
+            importMessage = "Upload complete";
             selectedFile = null;
             document.getElementById("import-file").value = "";
+            // Clear message after 5 seconds
+            setTimeout(() => { importMessage = ""; }, 5000);
         } catch (error) {
             console.error("Import failed:", error);
-            alert("Import failed: " + error.message);
+            importMessage = "Import failed: " + error.message;
+            setTimeout(() => { importMessage = ""; }, 5000);
         }
     }
 
@@ -2952,6 +2962,7 @@
                 {currentEffectiveRole}
                 {selectedFile}
                 {aclMode}
+                {importMessage}
                 on:fileSelect={handleFileSelect}
                 on:importEvents={importEvents}
                 on:openLoginModal={openLoginModal}

app/web/src/ImportView.svelte

@@ -3,6 +3,7 @@
     export let currentEffectiveRole = "";
     export let selectedFile = null;
     export let aclMode = "";
+    export let importMessage = "";
 
     import { createEventDispatcher } from "svelte";
     const dispatch = createEventDispatcher();
@@ -34,13 +35,18 @@
                 accept=".jsonl,.txt"
                 on:change={handleFileSelect}
             />
-            <button
-                class="import-btn"
-                on:click={importEvents}
-                disabled={!selectedFile}
-            >
-                Import Events
-            </button>
+            <div class="import-row">
+                <button
+                    class="import-btn"
+                    on:click={importEvents}
+                    disabled={!selectedFile || importMessage === "Uploading..."}
+                >
+                    Import Events
+                </button>
+                {#if importMessage}
+                    <span class="import-message" class:uploading={importMessage === "Uploading..."} class:success={importMessage === "Upload complete"} class:error={importMessage.startsWith("Import failed") || importMessage.startsWith("Admin") || importMessage.startsWith("Please")}>{importMessage}</span>
+                {/if}
+            </div>
         </div>
     {:else if isLoggedIn}
         <div class="permission-denied">
@@ -122,6 +128,30 @@
         cursor: not-allowed;
     }
 
+    .import-row {
+        display: flex;
+        align-items: center;
+        gap: 1em;
+    }
+
+    .import-message {
+        font-size: 0.9em;
+        padding: 0.25em 0.5em;
+        border-radius: 0.25em;
+    }
+
+    .import-message.uploading {
+        color: var(--primary);
+    }
+
+    .import-message.success {
+        color: #4caf50;
+    }
+
+    .import-message.error {
+        color: #f44336;
+    }
+
     .permission-denied,
     .login-prompt {
         text-align: center;

pkg/database/import.go

@@ -10,10 +10,11 @@ import (
 )
 
 // Import a collection of events in line structured minified JSON format (JSONL).
+// This runs synchronously to ensure the reader remains valid during processing.
+// The actual event processing happens after buffering to a temp file, so the
+// caller can close the reader after Import returns.
 func (d *D) Import(rr io.Reader) {
-	go func() {
-		if err := d.ImportEventsFromReader(d.ctx, rr); chk.E(err) {
-			log.E.F("import failed: %v", err)
-		}
-	}()
+	if err := d.ImportEventsFromReader(d.ctx, rr); chk.E(err) {
+		log.E.F("import failed: %v", err)
+	}
 }

pkg/database/migrations.go

@@ -850,66 +850,59 @@ func (d *D) ConvertToCompactEventFormat() {
 		return
 	}
 
-	// Second pass: convert in batches
-	const batchSize = 500
-	for i := 0; i < len(migrations); i += batchSize {
-		end := i + batchSize
-		if end > len(migrations) {
-			end = len(migrations)
-		}
-		batch := migrations[i:end]
-
+	// Process each event individually to avoid transaction size limits
+	// Some events (like kind 3 follow lists) can be very large
+	for i, m := range migrations {
 		if err = d.Update(func(txn *badger.Txn) error {
-			for _, m := range batch {
-				// Decode the legacy event
-				ev := new(event.E)
-				if err = ev.UnmarshalBinary(bytes.NewBuffer(m.OldData)); chk.E(err) {
-					log.W.F("migration: failed to decode event serial %d: %v", m.Serial, err)
-					continue
-				}
+			// Decode the legacy event
+			ev := new(event.E)
+			if err = ev.UnmarshalBinary(bytes.NewBuffer(m.OldData)); chk.E(err) {
+				log.W.F("migration: failed to decode event serial %d: %v", m.Serial, err)
+				return nil // Continue with next event
+			}
 
-				// Store SerialEventId mapping
-				if err = d.StoreEventIdSerial(txn, m.Serial, m.EventId); chk.E(err) {
-					log.W.F("migration: failed to store event ID mapping for serial %d: %v", m.Serial, err)
-					continue
-				}
+			// Store SerialEventId mapping
+			if err = d.StoreEventIdSerial(txn, m.Serial, m.EventId); chk.E(err) {
+				log.W.F("migration: failed to store event ID mapping for serial %d: %v", m.Serial, err)
+				return nil // Continue with next event
+			}
 
-				// Encode in compact format
-				compactData, encErr := MarshalCompactEvent(ev, resolver)
-				if encErr != nil {
-					log.W.F("migration: failed to encode compact event for serial %d: %v", m.Serial, encErr)
-					continue
-				}
+			// Encode in compact format
+			compactData, encErr := MarshalCompactEvent(ev, resolver)
+			if encErr != nil {
+				log.W.F("migration: failed to encode compact event for serial %d: %v", m.Serial, encErr)
+				return nil // Continue with next event
+			}
 
-				// Store compact event
-				ser := new(types.Uint40)
-				if err = ser.Set(m.Serial); chk.E(err) {
-					continue
-				}
-				cmpKey := new(bytes.Buffer)
-				if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKey); chk.E(err) {
-					continue
-				}
-				if err = txn.Set(cmpKey.Bytes(), compactData); chk.E(err) {
-					log.W.F("migration: failed to store compact event for serial %d: %v", m.Serial, err)
-					continue
-				}
+			// Store compact event
+			ser := new(types.Uint40)
+			if err = ser.Set(m.Serial); chk.E(err) {
+				return nil // Continue with next event
+			}
+			cmpKey := new(bytes.Buffer)
+			if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKey); chk.E(err) {
+				return nil // Continue with next event
+			}
+			if err = txn.Set(cmpKey.Bytes(), compactData); chk.E(err) {
+				log.W.F("migration: failed to store compact event for serial %d: %v", m.Serial, err)
+				return nil // Continue with next event
+			}
 
-				// Track savings
-				savedBytes += int64(len(m.OldData) - len(compactData))
-				processedCount++
+			// Track savings
+			savedBytes += int64(len(m.OldData) - len(compactData))
+			processedCount++
 
-				// Cache the mappings
-				d.serialCache.CacheEventId(m.Serial, m.EventId)
-			}
+			// Cache the mappings
+			d.serialCache.CacheEventId(m.Serial, m.EventId)
 			return nil
 		}); chk.E(err) {
-			log.W.F("batch migration failed: %v", err)
+			log.W.F("migration failed for event %d: %v", m.Serial, err)
 			continue
 		}
 
-		if (i/batchSize)%10 == 0 && i > 0 {
-			log.I.F("migration progress: %d/%d events converted", i, len(migrations))
+		// Log progress every 1000 events
+		if (i+1)%1000 == 0 {
+			log.I.F("migration progress: %d/%d events converted", i+1, len(migrations))
 		}
 	}
 

pkg/version/version

@@ -1 +1 @@
-v0.34.3
+v0.34.4