Update event import process and improve user feedback

Simplified event import to run synchronously, ensuring proper resource handling and accurate feedback. Enhanced frontend with real-time import status messages and error handling. Adjusted migrations to handle events individually, improving reliability and granular progress tracking.

.claude/settings.local.json

@@ -12,7 +12,8 @@
       "Write:*",
       "Bash(go build:*)",
       "Bash(go test:*)",
-      "Bash(./scripts/test.sh:*)"
+      "Bash(./scripts/test.sh:*)",
+      "Bash(./scripts/update-embedded-web.sh:*)"
     ],
     "deny": [],
     "ask": []

app/web/src/App.svelte

@@ -53,6 +53,7 @@
     let searchTabs = [];
     let allEvents = [];
     let selectedFile = null;
+    let importMessage = ""; // Message shown after import completes
     let expandedEvents = new Set();
     let isLoadingEvents = false;
     let hasMoreEvents = true;
@@ -2356,22 +2357,28 @@
 
     // Import functionality
     function handleFileSelect(event) {
-        selectedFile = event.target.files[0];
+        // event.detail contains the original DOM event from the child component
+        selectedFile = event.detail.target.files[0];
     }
 
     async function importEvents() {
         // Skip login/permission check when ACL is "none" (open relay mode)
         if (aclMode !== "none" && (!isLoggedIn || (userRole !== "admin" && userRole !== "owner"))) {
-            alert("Admin or owner permission required");
+            importMessage = "Admin or owner permission required";
+            setTimeout(() => { importMessage = ""; }, 5000);
             return;
         }
 
         if (!selectedFile) {
-            alert("Please select a file");
+            importMessage = "Please select a file";
+            setTimeout(() => { importMessage = ""; }, 5000);
             return;
         }
 
         try {
+            // Show uploading message
+            importMessage = "Uploading...";
+
             // Build headers - only include auth when ACL is not "none"
             const headers = {};
             if (aclMode !== "none" && isLoggedIn) {
@@ -2396,12 +2403,15 @@
             }
 
             const result = await response.json();
-            alert("Import started successfully");
+            importMessage = "Upload complete";
             selectedFile = null;
             document.getElementById("import-file").value = "";
+            // Clear message after 5 seconds
+            setTimeout(() => { importMessage = ""; }, 5000);
         } catch (error) {
             console.error("Import failed:", error);
-            alert("Import failed: " + error.message);
+            importMessage = "Import failed: " + error.message;
+            setTimeout(() => { importMessage = ""; }, 5000);
         }
     }
 
@@ -2952,6 +2962,7 @@
                 {currentEffectiveRole}
                 {selectedFile}
                 {aclMode}
+                {importMessage}
                 on:fileSelect={handleFileSelect}
                 on:importEvents={importEvents}
                 on:openLoginModal={openLoginModal}

app/web/src/ImportView.svelte

@@ -3,6 +3,7 @@
     export let currentEffectiveRole = "";
     export let selectedFile = null;
     export let aclMode = "";
+    export let importMessage = "";
 
     import { createEventDispatcher } from "svelte";
     const dispatch = createEventDispatcher();
@@ -34,13 +35,18 @@
                 accept=".jsonl,.txt"
                 on:change={handleFileSelect}
             />
+            <div class="import-row">
                 <button
                     class="import-btn"
                     on:click={importEvents}
-                disabled={!selectedFile}
+                    disabled={!selectedFile || importMessage === "Uploading..."}
                 >
                     Import Events
                 </button>
+                {#if importMessage}
+                    <span class="import-message" class:uploading={importMessage === "Uploading..."} class:success={importMessage === "Upload complete"} class:error={importMessage.startsWith("Import failed") || importMessage.startsWith("Admin") || importMessage.startsWith("Please")}>{importMessage}</span>
+                {/if}
+            </div>
         </div>
     {:else if isLoggedIn}
         <div class="permission-denied">
@@ -122,6 +128,30 @@
         cursor: not-allowed;
     }
 
+    .import-row {
+        display: flex;
+        align-items: center;
+        gap: 1em;
+    }
+
+    .import-message {
+        font-size: 0.9em;
+        padding: 0.25em 0.5em;
+        border-radius: 0.25em;
+    }
+
+    .import-message.uploading {
+        color: var(--primary);
+    }
+
+    .import-message.success {
+        color: #4caf50;
+    }
+
+    .import-message.error {
+        color: #f44336;
+    }
+
     .permission-denied,
     .login-prompt {
         text-align: center;

pkg/database/import.go

@@ -10,10 +10,11 @@ import (
 )
 
 // Import a collection of events in line structured minified JSON format (JSONL).
+// This runs synchronously to ensure the reader remains valid during processing.
+// The actual event processing happens after buffering to a temp file, so the
+// caller can close the reader after Import returns.
 func (d *D) Import(rr io.Reader) {
-	go func() {
 	if err := d.ImportEventsFromReader(d.ctx, rr); chk.E(err) {
 		log.E.F("import failed: %v", err)
 	}
-	}()
 }

pkg/database/migrations.go

@@ -850,49 +850,42 @@ func (d *D) ConvertToCompactEventFormat() {
 		return
 	}
 
-	// Second pass: convert in batches
-	const batchSize = 500
-	for i := 0; i < len(migrations); i += batchSize {
-		end := i + batchSize
-		if end > len(migrations) {
-			end = len(migrations)
-		}
-		batch := migrations[i:end]
-
+	// Process each event individually to avoid transaction size limits
+	// Some events (like kind 3 follow lists) can be very large
+	for i, m := range migrations {
 		if err = d.Update(func(txn *badger.Txn) error {
-			for _, m := range batch {
 			// Decode the legacy event
 			ev := new(event.E)
 			if err = ev.UnmarshalBinary(bytes.NewBuffer(m.OldData)); chk.E(err) {
 				log.W.F("migration: failed to decode event serial %d: %v", m.Serial, err)
-					continue
+				return nil // Continue with next event
 			}
 
 			// Store SerialEventId mapping
 			if err = d.StoreEventIdSerial(txn, m.Serial, m.EventId); chk.E(err) {
 				log.W.F("migration: failed to store event ID mapping for serial %d: %v", m.Serial, err)
-					continue
+				return nil // Continue with next event
 			}
 
 			// Encode in compact format
 			compactData, encErr := MarshalCompactEvent(ev, resolver)
 			if encErr != nil {
 				log.W.F("migration: failed to encode compact event for serial %d: %v", m.Serial, encErr)
-					continue
+				return nil // Continue with next event
 			}
 
 			// Store compact event
 			ser := new(types.Uint40)
 			if err = ser.Set(m.Serial); chk.E(err) {
-					continue
+				return nil // Continue with next event
 			}
 			cmpKey := new(bytes.Buffer)
 			if err = indexes.CompactEventEnc(ser).MarshalWrite(cmpKey); chk.E(err) {
-					continue
+				return nil // Continue with next event
 			}
 			if err = txn.Set(cmpKey.Bytes(), compactData); chk.E(err) {
 				log.W.F("migration: failed to store compact event for serial %d: %v", m.Serial, err)
-					continue
+				return nil // Continue with next event
 			}
 
 			// Track savings
@@ -901,15 +894,15 @@ func (d *D) ConvertToCompactEventFormat() {
 
 			// Cache the mappings
 			d.serialCache.CacheEventId(m.Serial, m.EventId)
-			}
 			return nil
 		}); chk.E(err) {
-			log.W.F("batch migration failed: %v", err)
+			log.W.F("migration failed for event %d: %v", m.Serial, err)
 			continue
 		}
 
-		if (i/batchSize)%10 == 0 && i > 0 {
-			log.I.F("migration progress: %d/%d events converted", i, len(migrations))
+		// Log progress every 1000 events
+		if (i+1)%1000 == 0 {
+			log.I.F("migration progress: %d/%d events converted", i+1, len(migrations))
 		}
 	}
 

pkg/version/version

@@ -1 +1 @@
-v0.34.3
+v0.34.4