6 changed files with 450 additions and 0 deletions
@ -0,0 +1,218 @@
@@ -0,0 +1,218 @@
|
||||
import { Button } from '@/components/ui/button' |
||||
import { Checkbox } from '@/components/ui/checkbox' |
||||
import { Label } from '@/components/ui/label' |
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' |
||||
import { Textarea } from '@/components/ui/textarea' |
||||
import { |
||||
CACHE_IMPORT_MAX_JSONL_FILE_BYTES, |
||||
formatCacheImportLimits, |
||||
parseJsonlCacheImportText, |
||||
parsePastedCacheImportJson |
||||
} from '@/lib/cache-event-import' |
||||
import { showPublishingError, showSimplePublishSuccess } from '@/lib/publishing-feedback' |
||||
import { useNostr } from '@/providers/NostrProvider' |
||||
import client from '@/services/client.service' |
||||
import type { Event } from 'nostr-tools' |
||||
import { Upload } from 'lucide-react' |
||||
import { useCallback, useId, useRef, useState } from 'react' |
||||
import { useTranslation } from 'react-i18next' |
||||
import { toast } from 'sonner' |
||||
|
||||
async function ingestImportedEvents( |
||||
events: Event[], |
||||
broadcast: boolean, |
||||
accountPubkey: string | null, |
||||
t: (key: string) => string |
||||
): Promise<{ imported: number; broadcastOk: number; broadcastFailed: number }> { |
||||
const unique: Event[] = [] |
||||
const seen = new Set<string>() |
||||
for (const ev of events) { |
||||
const id = ev.id.toLowerCase() |
||||
if (seen.has(id)) continue |
||||
seen.add(id) |
||||
unique.push(ev) |
||||
client.addEventToCache(ev, { explicitNoteLookupHexId: id }) |
||||
} |
||||
|
||||
let broadcastOk = 0 |
||||
let broadcastFailed = 0 |
||||
if (broadcast && accountPubkey && unique.length > 0) { |
||||
const urls = await client.getMailboxStackWriteUrlsForRepublish(accountPubkey) |
||||
if (!urls.length) { |
||||
throw new Error(t('No mailbox cache or HTTP write relays configured')) |
||||
} |
||||
for (const ev of unique) { |
||||
try { |
||||
const result = await client.publishEvent(urls, ev, { skipOutboxRetry: true }) |
||||
if (result.successCount >= 1) broadcastOk++ |
||||
else broadcastFailed++ |
||||
} catch { |
||||
broadcastFailed++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
return { imported: unique.length, broadcastOk, broadcastFailed } |
||||
} |
||||
|
||||
export default function CacheEventImportSettings() { |
||||
const { t } = useTranslation() |
||||
const { pubkey: accountPubkey } = useNostr() |
||||
const broadcastId = useId() |
||||
const fileInputRef = useRef<HTMLInputElement>(null) |
||||
const [pasteText, setPasteText] = useState('') |
||||
const [broadcastAfterImport, setBroadcastAfterImport] = useState(false) |
||||
const [busy, setBusy] = useState(false) |
||||
|
||||
const runImport = useCallback( |
||||
async (events: Event[], issueCount: number) => { |
||||
if (events.length === 0) { |
||||
toast.error(t('cacheImport.noValidEvents')) |
||||
return |
||||
} |
||||
if (broadcastAfterImport && !accountPubkey) { |
||||
toast.error(t('Log in to broadcast')) |
||||
return |
||||
} |
||||
setBusy(true) |
||||
const loadingId = toast.loading(t('cacheImport.importing')) |
||||
try { |
||||
const stats = await ingestImportedEvents(events, broadcastAfterImport, accountPubkey ?? null, t) |
||||
if (issueCount > 0) { |
||||
toast.warning( |
||||
t('cacheImport.partialWithIssues', { |
||||
imported: stats.imported, |
||||
skipped: issueCount |
||||
}) |
||||
) |
||||
} |
||||
const successMessage = |
||||
broadcastAfterImport && accountPubkey |
||||
? t('cacheImport.doneWithBroadcast', { |
||||
imported: stats.imported, |
||||
broadcastOk: stats.broadcastOk, |
||||
broadcastFailed: stats.broadcastFailed |
||||
}) |
||||
: t('cacheImport.done', { count: stats.imported }) |
||||
toast.dismiss(loadingId) |
||||
showSimplePublishSuccess(successMessage) |
||||
setPasteText('') |
||||
} catch (err) { |
||||
toast.dismiss(loadingId) |
||||
showPublishingError( |
||||
t('cacheImport.failed', { |
||||
error: err instanceof Error ? err.message : String(err) |
||||
}) |
||||
) |
||||
} finally { |
||||
setBusy(false) |
||||
} |
||||
}, |
||||
[accountPubkey, broadcastAfterImport, t] |
||||
) |
||||
|
||||
const handleImportPaste = () => { |
||||
const { events, issues } = parsePastedCacheImportJson(pasteText) |
||||
void runImport(events, issues.length) |
||||
} |
||||
|
||||
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => { |
||||
const file = e.target.files?.[0] |
||||
e.target.value = '' |
||||
if (!file) return |
||||
if (!file.name.toLowerCase().endsWith('.jsonl')) { |
||||
toast.error(t('cacheImport.jsonlOnly')) |
||||
return |
||||
} |
||||
if (file.size > CACHE_IMPORT_MAX_JSONL_FILE_BYTES) { |
||||
toast.error( |
||||
t('cacheImport.fileTooLarge', { |
||||
mb: Math.round(CACHE_IMPORT_MAX_JSONL_FILE_BYTES / (1024 * 1024)) |
||||
}) |
||||
) |
||||
return |
||||
} |
||||
void (async () => { |
||||
setBusy(true) |
||||
try { |
||||
const text = await file.text() |
||||
const { events, issues } = parseJsonlCacheImportText(text) |
||||
await runImport(events, issues.length) |
||||
} catch (err) { |
||||
toast.error( |
||||
t('cacheImport.failed', { |
||||
error: err instanceof Error ? err.message : String(err) |
||||
}) |
||||
) |
||||
} finally { |
||||
setBusy(false) |
||||
} |
||||
})() |
||||
} |
||||
|
||||
return ( |
||||
<div className="space-y-4 border-t border-border pt-6"> |
||||
<h3 className="text-base font-medium">{t('cacheImport.sectionTitle')}</h3> |
||||
<p className="text-sm text-muted-foreground">{t('cacheImport.sectionBlurb')}</p> |
||||
<p className="text-xs text-muted-foreground">{formatCacheImportLimits()}</p> |
||||
|
||||
<div className="flex items-start gap-2"> |
||||
<Checkbox |
||||
id={broadcastId} |
||||
checked={broadcastAfterImport} |
||||
onCheckedChange={(v) => setBroadcastAfterImport(v === true)} |
||||
disabled={!accountPubkey || busy} |
||||
/> |
||||
<Label htmlFor={broadcastId} className="text-sm font-normal leading-snug cursor-pointer"> |
||||
{t('cacheImport.broadcastLabel')} |
||||
{!accountPubkey ? ( |
||||
<span className="block text-xs text-muted-foreground">{t('Log in to broadcast')}</span> |
||||
) : null} |
||||
</Label> |
||||
</div> |
||||
|
||||
<Tabs defaultValue="paste" className="w-full"> |
||||
<TabsList className="w-full"> |
||||
<TabsTrigger value="paste" className="flex-1"> |
||||
{t('cacheImport.tabPaste')} |
||||
</TabsTrigger> |
||||
<TabsTrigger value="file" className="flex-1"> |
||||
{t('cacheImport.tabFile')} |
||||
</TabsTrigger> |
||||
</TabsList> |
||||
<TabsContent value="paste" className="space-y-3 mt-3"> |
||||
<Textarea |
||||
className="min-h-[140px] font-mono text-xs" |
||||
placeholder={t('cacheImport.pastePlaceholder')} |
||||
value={pasteText} |
||||
onChange={(e) => setPasteText(e.target.value)} |
||||
disabled={busy} |
||||
/> |
||||
<Button type="button" disabled={busy || !pasteText.trim()} onClick={handleImportPaste}> |
||||
{t('cacheImport.importButton')} |
||||
</Button> |
||||
</TabsContent> |
||||
<TabsContent value="file" className="space-y-3 mt-3"> |
||||
<input |
||||
ref={fileInputRef} |
||||
type="file" |
||||
accept=".jsonl,application/jsonl,text/jsonl" |
||||
className="hidden" |
||||
onChange={handleFileChange} |
||||
disabled={busy} |
||||
/> |
||||
<Button |
||||
type="button" |
||||
variant="secondary" |
||||
disabled={busy} |
||||
onClick={() => fileInputRef.current?.click()} |
||||
> |
||||
<Upload className="mr-2 h-4 w-4" /> |
||||
{t('cacheImport.chooseJsonl')} |
||||
</Button> |
||||
<p className="text-xs text-muted-foreground">{t('cacheImport.fileHint')}</p> |
||||
</TabsContent> |
||||
</Tabs> |
||||
</div> |
||||
) |
||||
} |
||||
@ -0,0 +1,44 @@
@@ -0,0 +1,44 @@
|
||||
import { describe, expect, it } from 'vitest' |
||||
import { kinds, finalizeEvent, generateSecretKey, getPublicKey } from 'nostr-tools' |
||||
import { parseJsonlCacheImportText, parsePastedCacheImportJson } from './cache-event-import' |
||||
|
||||
function signedNote(content: string) { |
||||
const sk = generateSecretKey() |
||||
const pubkey = getPublicKey(sk) |
||||
return finalizeEvent( |
||||
{ |
||||
kind: kinds.ShortTextNote, |
||||
created_at: Math.floor(Date.now() / 1000), |
||||
tags: [], |
||||
content |
||||
}, |
||||
sk |
||||
) |
||||
} |
||||
|
||||
describe('parsePastedCacheImportJson', () => { |
||||
it('accepts a single signed event object', () => { |
||||
const ev = signedNote('hello') |
||||
const { events, issues } = parsePastedCacheImportJson(JSON.stringify(ev)) |
||||
expect(issues).toHaveLength(0) |
||||
expect(events).toHaveLength(1) |
||||
expect(events[0].id).toBe(ev.id) |
||||
}) |
||||
|
||||
it('rejects invalid JSON', () => { |
||||
const { events, issues } = parsePastedCacheImportJson('{not json') |
||||
expect(events).toHaveLength(0) |
||||
expect(issues[0].message).toMatch(/Invalid JSON/i) |
||||
}) |
||||
}) |
||||
|
||||
describe('parseJsonlCacheImportText', () => { |
||||
it('parses one event per line', () => { |
||||
const a = signedNote('a') |
||||
const b = signedNote('b') |
||||
const text = `${JSON.stringify(a)}\n${JSON.stringify(b)}\n` |
||||
const { events, issues } = parseJsonlCacheImportText(text) |
||||
expect(issues).toHaveLength(0) |
||||
expect(events).toHaveLength(2) |
||||
}) |
||||
}) |
||||
@ -0,0 +1,148 @@
@@ -0,0 +1,148 @@
|
||||
import { isLikelyCachedNostrEvent } from '@/services/indexed-db.service' |
||||
import type { Event } from 'nostr-tools' |
||||
import { validateEvent, verifyEvent } from 'nostr-tools' |
||||
|
||||
/** Max size for a pasted JSON blob (single event or small array). */ |
||||
export const CACHE_IMPORT_MAX_PASTE_CHARS = 400_000 |
||||
|
||||
/** Max `.jsonl` upload size. */ |
||||
export const CACHE_IMPORT_MAX_JSONL_FILE_BYTES = 2 * 1024 * 1024 |
||||
|
||||
/** Max lines processed from a `.jsonl` file. */ |
||||
export const CACHE_IMPORT_MAX_JSONL_LINES = 500 |
||||
|
||||
/** Skip oversized single lines in JSONL. */ |
||||
export const CACHE_IMPORT_MAX_JSONL_LINE_CHARS = 256_000 |
||||
|
||||
export type CacheImportParseIssue = { line?: number; message: string } |
||||
|
||||
export type CacheImportParseResult = { |
||||
events: Event[] |
||||
issues: CacheImportParseIssue[] |
||||
} |
||||
|
||||
function normalizeImportedEvent(raw: unknown): Event | null { |
||||
if (!isLikelyCachedNostrEvent(raw)) return null |
||||
const ev = raw as Event |
||||
if (typeof ev.created_at !== 'number' || !Number.isFinite(ev.created_at)) return null |
||||
if (typeof ev.sig !== 'string' || !ev.sig.trim()) return null |
||||
if (!validateEvent(ev)) return null |
||||
if (!verifyEvent(ev)) return null |
||||
return ev |
||||
} |
||||
|
||||
function collectFromUnknown( |
||||
value: unknown, |
||||
line: number | undefined, |
||||
events: Event[], |
||||
issues: CacheImportParseIssue[] |
||||
): void { |
||||
if (Array.isArray(value)) { |
||||
if (value.length + events.length > CACHE_IMPORT_MAX_JSONL_LINES) { |
||||
issues.push({ |
||||
line, |
||||
message: `Too many events (max ${CACHE_IMPORT_MAX_JSONL_LINES})` |
||||
}) |
||||
return |
||||
} |
||||
for (let i = 0; i < value.length; i++) { |
||||
collectFromUnknown(value[i], line, events, issues) |
||||
if (events.length >= CACHE_IMPORT_MAX_JSONL_LINES) break |
||||
} |
||||
return |
||||
} |
||||
const ev = normalizeImportedEvent(value) |
||||
if (ev) { |
||||
events.push(ev) |
||||
return |
||||
} |
||||
issues.push({ |
||||
line, |
||||
message: line != null ? 'Invalid or unsigned event on this line' : 'Invalid or unsigned Nostr event JSON' |
||||
}) |
||||
} |
||||
|
||||
/** Parse one pasted JSON value (object or array of events). */ |
||||
export function parsePastedCacheImportJson(text: string): CacheImportParseResult { |
||||
const trimmed = text.trim() |
||||
if (!trimmed) { |
||||
return { events: [], issues: [{ message: 'Paste is empty' }] } |
||||
} |
||||
if (trimmed.length > CACHE_IMPORT_MAX_PASTE_CHARS) { |
||||
return { |
||||
events: [], |
||||
issues: [ |
||||
{ |
||||
message: `Paste exceeds ${Math.round(CACHE_IMPORT_MAX_PASTE_CHARS / 1024)} KB limit` |
||||
} |
||||
] |
||||
} |
||||
} |
||||
let parsed: unknown |
||||
try { |
||||
parsed = JSON.parse(trimmed) as unknown |
||||
} catch { |
||||
return { events: [], issues: [{ message: 'Invalid JSON' }] } |
||||
} |
||||
const events: Event[] = [] |
||||
const issues: CacheImportParseIssue[] = [] |
||||
collectFromUnknown(parsed, undefined, events, issues) |
||||
return { events, issues } |
||||
} |
||||
|
||||
/** Parse newline-delimited JSON (one event per line). */ |
||||
export function parseJsonlCacheImportText(text: string): CacheImportParseResult { |
||||
if (text.length > CACHE_IMPORT_MAX_JSONL_FILE_BYTES) { |
||||
return { |
||||
events: [], |
||||
issues: [ |
||||
{ |
||||
message: `File content exceeds ${Math.round(CACHE_IMPORT_MAX_JSONL_FILE_BYTES / (1024 * 1024))} MB limit` |
||||
} |
||||
] |
||||
} |
||||
} |
||||
const lines = text.split(/\r?\n/) |
||||
const events: Event[] = [] |
||||
const issues: CacheImportParseIssue[] = [] |
||||
let nonEmptyLines = 0 |
||||
for (let i = 0; i < lines.length; i++) { |
||||
const line = lines[i].trim() |
||||
if (!line) continue |
||||
nonEmptyLines++ |
||||
if (nonEmptyLines > CACHE_IMPORT_MAX_JSONL_LINES) { |
||||
issues.push({ |
||||
line: i + 1, |
||||
message: `Stopped at ${CACHE_IMPORT_MAX_JSONL_LINES} events (file has more lines)` |
||||
}) |
||||
break |
||||
} |
||||
if (line.length > CACHE_IMPORT_MAX_JSONL_LINE_CHARS) { |
||||
issues.push({ line: i + 1, message: 'Line too large' }) |
||||
continue |
||||
} |
||||
let parsed: unknown |
||||
try { |
||||
parsed = JSON.parse(line) as unknown |
||||
} catch { |
||||
issues.push({ line: i + 1, message: 'Invalid JSON on this line' }) |
||||
continue |
||||
} |
||||
const before = events.length |
||||
collectFromUnknown(parsed, i + 1, events, issues) |
||||
if (events.length === before && issues[issues.length - 1]?.line !== i + 1) { |
||||
issues.push({ line: i + 1, message: 'Invalid or unsigned event on this line' }) |
||||
} |
||||
if (events.length >= CACHE_IMPORT_MAX_JSONL_LINES) break |
||||
} |
||||
if (nonEmptyLines === 0) { |
||||
issues.push({ message: 'File has no event lines' }) |
||||
} |
||||
return { events, issues } |
||||
} |
||||
|
||||
export function formatCacheImportLimits(): string { |
||||
const mb = Math.round(CACHE_IMPORT_MAX_JSONL_FILE_BYTES / (1024 * 1024)) |
||||
const pasteKb = Math.round(CACHE_IMPORT_MAX_PASTE_CHARS / 1024) |
||||
return `${CACHE_IMPORT_MAX_JSONL_LINES} events max · ${mb} MB file · ${pasteKb} KB paste` |
||||
} |
||||
Loading…
Reference in new issue