Browse Source
Replace manual regex parsing with proper Asciidoctor AST traversal for more accurate and scalable document processing. Add dynamic parse level system that correctly maps AsciiDoc hierarchy to Nostr events. Key changes: - New asciidoc_ast_parser.ts using Asciidoctor's native document structure - PublicationTree factory for seamless integration with existing architecture - Fixed level mapping: Asciidoctor levels 1-5 → app levels 2-6 - Dynamic parse level generation (2-6) with proper event count progression - Responsive editor height using viewport units for adaptive screen sizing - Enhanced preview with hierarchical visualization and event boundaries Event count progression now works correctly: - Level 2: 3 events → Level 6: 20+ events - Full reactivity when changing parse levels - Proper content type detection (Article vs Notes vs None) 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>master
4 changed files with 713 additions and 83 deletions
@ -0,0 +1,273 @@
@@ -0,0 +1,273 @@
|
||||
/** |
||||
* AST-based AsciiDoc parsing using Asciidoctor's native document structure |
||||
*
|
||||
* This replaces the manual regex parsing in asciidoc_metadata.ts with proper |
||||
* AST traversal, leveraging Asciidoctor's built-in parsing capabilities. |
||||
*/ |
||||
|
||||
import Processor from "asciidoctor"; |
||||
import type { Document } from "asciidoctor"; |
||||
import { PublicationTree } from "../data_structures/publication_tree.ts"; |
||||
import { NDKEvent } from "@nostr-dev-kit/ndk"; |
||||
import type NDK from "@nostr-dev-kit/ndk"; |
||||
import { getMimeTags } from "./mime.ts"; |
||||
|
||||
export interface ASTSection { |
||||
title: string; |
||||
content: string; |
||||
level: number; |
||||
attributes: Record<string, string>; |
||||
subsections: ASTSection[]; |
||||
} |
||||
|
||||
export interface ASTParsedDocument { |
||||
title: string; |
||||
content: string; |
||||
attributes: Record<string, string>; |
||||
sections: ASTSection[]; |
||||
} |
||||
|
||||
/** |
||||
* Parse AsciiDoc content using Asciidoctor's AST instead of manual regex |
||||
*/ |
||||
export function parseAsciiDocAST(content: string, parseLevel: number = 2): ASTParsedDocument { |
||||
const asciidoctor = Processor(); |
||||
const document = asciidoctor.load(content, { standalone: false }) as Document; |
||||
|
||||
return { |
||||
title: document.getTitle() || '', |
||||
content: document.getContent() || '', |
||||
attributes: document.getAttributes(), |
||||
sections: extractSectionsFromAST(document, parseLevel) |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Extract sections from Asciidoctor AST based on parse level |
||||
*/ |
||||
function extractSectionsFromAST(document: Document, parseLevel: number): ASTSection[] { |
||||
const directSections = document.getSections(); |
||||
|
||||
// Collect all sections at all levels up to parseLevel
|
||||
const allSections: ASTSection[] = []; |
||||
|
||||
function collectSections(sections: any[]) { |
||||
for (const section of sections) { |
||||
const asciidoctorLevel = section.getLevel(); |
||||
// Convert Asciidoctor's internal level to our application level
|
||||
// Asciidoctor: == is level 1, === is level 2, etc.
|
||||
// Our app: == is level 2, === is level 3, etc.
|
||||
const appLevel = asciidoctorLevel + 1; |
||||
|
||||
if (appLevel <= parseLevel) { |
||||
allSections.push({ |
||||
title: section.getTitle() || '', |
||||
content: section.getContent() || '', |
||||
level: appLevel, |
||||
attributes: section.getAttributes() || {}, |
||||
subsections: [] |
||||
}); |
||||
} |
||||
|
||||
// Recursively collect subsections
|
||||
const subsections = section.getSections?.() || []; |
||||
if (subsections.length > 0) { |
||||
collectSections(subsections); |
||||
} |
||||
} |
||||
} |
||||
|
||||
collectSections(directSections); |
||||
|
||||
return allSections; |
||||
} |
||||
|
||||
/** |
||||
* Extract subsections from a section (recursive helper) |
||||
*/ |
||||
function extractSubsections(section: any, parseLevel: number): ASTSection[] { |
||||
const subsections = section.getSections?.() || []; |
||||
|
||||
return subsections |
||||
.filter((sub: any) => (sub.getLevel() + 1) <= parseLevel) |
||||
.map((sub: any) => ({ |
||||
title: sub.getTitle() || '', |
||||
content: sub.getContent() || '', |
||||
level: sub.getLevel() + 1, // Convert to app level
|
||||
attributes: sub.getAttributes() || {}, |
||||
subsections: extractSubsections(sub, parseLevel) |
||||
})); |
||||
} |
||||
|
||||
/** |
||||
* Create a PublicationTree directly from Asciidoctor AST |
||||
* This integrates with Michael's PublicationTree architecture |
||||
*/ |
||||
export async function createPublicationTreeFromAST( |
||||
content: string,
|
||||
ndk: NDK,
|
||||
parseLevel: number = 2 |
||||
): Promise<PublicationTree> { |
||||
const parsed = parseAsciiDocAST(content, parseLevel); |
||||
|
||||
// Create root 30040 index event from document metadata
|
||||
const rootEvent = createIndexEventFromAST(parsed, ndk); |
||||
const tree = new PublicationTree(rootEvent, ndk); |
||||
|
||||
// Add sections as 30041 events
|
||||
for (const section of parsed.sections) { |
||||
const contentEvent = createContentEventFromSection(section, ndk); |
||||
await tree.addEvent(contentEvent, rootEvent); |
||||
} |
||||
|
||||
return tree; |
||||
} |
||||
|
||||
/** |
||||
* Create a 30040 index event from AST document metadata |
||||
*/ |
||||
function createIndexEventFromAST(parsed: ASTParsedDocument, ndk: NDK): NDKEvent { |
||||
const event = new NDKEvent(ndk); |
||||
event.kind = 30040; |
||||
event.created_at = Math.floor(Date.now() / 1000); |
||||
|
||||
// Generate d-tag from title
|
||||
const dTag = generateDTag(parsed.title); |
||||
const [mTag, MTag] = getMimeTags(30040); |
||||
|
||||
const tags: string[][] = [ |
||||
["d", dTag], |
||||
mTag, |
||||
MTag, |
||||
["title", parsed.title] |
||||
]; |
||||
|
||||
// Add document attributes as tags
|
||||
addAttributesAsTags(tags, parsed.attributes); |
||||
|
||||
// Add a-tags for each section (30041 content events)
|
||||
parsed.sections.forEach(section => { |
||||
const sectionDTag = generateDTag(section.title); |
||||
tags.push(["a", `30041:${ndk.activeUser?.pubkey || 'pubkey'}:${sectionDTag}`]); |
||||
}); |
||||
|
||||
event.tags = tags; |
||||
event.content = parsed.content; |
||||
|
||||
return event; |
||||
} |
||||
|
||||
/** |
||||
* Create a 30041 content event from an AST section |
||||
*/ |
||||
function createContentEventFromSection(section: ASTSection, ndk: NDK): NDKEvent { |
||||
const event = new NDKEvent(ndk); |
||||
event.kind = 30041; |
||||
event.created_at = Math.floor(Date.now() / 1000); |
||||
|
||||
const dTag = generateDTag(section.title); |
||||
const [mTag, MTag] = getMimeTags(30041); |
||||
|
||||
const tags: string[][] = [ |
||||
["d", dTag], |
||||
mTag, |
||||
MTag, |
||||
["title", section.title] |
||||
]; |
||||
|
||||
// Add section attributes as tags
|
||||
addAttributesAsTags(tags, section.attributes); |
||||
|
||||
event.tags = tags; |
||||
event.content = section.content; |
||||
|
||||
return event; |
||||
} |
||||
|
||||
/** |
||||
* Generate a deterministic d-tag from title |
||||
*/ |
||||
function generateDTag(title: string): string { |
||||
return title |
||||
.toLowerCase() |
||||
.replace(/[^\p{L}\p{N}]/gu, "-") |
||||
.replace(/-+/g, "-") |
||||
.replace(/^-|-$/g, ""); |
||||
} |
||||
|
||||
/** |
||||
* Add AsciiDoc attributes as Nostr event tags, filtering out system attributes |
||||
*/ |
||||
function addAttributesAsTags(tags: string[][], attributes: Record<string, string>) { |
||||
const systemAttributes = [ |
||||
'attribute-undefined', 'attribute-missing', 'appendix-caption', 'appendix-refsig', |
||||
'caution-caption', 'chapter-refsig', 'example-caption', 'figure-caption', |
||||
'important-caption', 'last-update-label', 'manname-title', 'note-caption', |
||||
'part-refsig', 'preface-title', 'section-refsig', 'table-caption', |
||||
'tip-caption', 'toc-title', 'untitled-label', 'version-label', 'warning-caption', |
||||
'asciidoctor', 'asciidoctor-version', 'safe-mode-name', 'backend', 'doctype', |
||||
'basebackend', 'filetype', 'outfilesuffix', 'stylesdir', 'iconsdir', |
||||
'localdate', 'localyear', 'localtime', 'localdatetime', 'docdate', |
||||
'docyear', 'doctime', 'docdatetime', 'doctitle', 'embedded', 'notitle' |
||||
]; |
||||
|
||||
// Add standard metadata tags
|
||||
if (attributes.author) tags.push(["author", attributes.author]); |
||||
if (attributes.version) tags.push(["version", attributes.version]); |
||||
if (attributes.description) tags.push(["summary", attributes.description]); |
||||
if (attributes.tags) { |
||||
attributes.tags.split(',').forEach(tag =>
|
||||
tags.push(["t", tag.trim()]) |
||||
); |
||||
} |
||||
|
||||
// Add custom attributes (non-system)
|
||||
Object.entries(attributes).forEach(([key, value]) => { |
||||
if (!systemAttributes.includes(key) && value) { |
||||
tags.push([key, value]); |
||||
} |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Tree processor extension for Asciidoctor |
||||
* This can be registered to automatically populate PublicationTree during parsing |
||||
*/ |
||||
export function createPublicationTreeProcessor(ndk: NDK, parseLevel: number = 2) { |
||||
return function(extensions: any) { |
||||
extensions.treeProcessor(function(this: any) { |
||||
const dsl = this; |
||||
dsl.process(function(this: any, document: Document) { |
||||
// Create PublicationTree and store on document for later retrieval
|
||||
const publicationTree = createPublicationTreeFromDocument(document, ndk, parseLevel); |
||||
document.setAttribute('publicationTree', publicationTree); |
||||
}); |
||||
}); |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Helper function to create PublicationTree from Asciidoctor Document |
||||
*/ |
||||
async function createPublicationTreeFromDocument( |
||||
document: Document,
|
||||
ndk: NDK,
|
||||
parseLevel: number |
||||
): Promise<PublicationTree> { |
||||
const parsed: ASTParsedDocument = { |
||||
title: document.getTitle() || '', |
||||
content: document.getContent() || '', |
||||
attributes: document.getAttributes(), |
||||
sections: extractSectionsFromAST(document, parseLevel) |
||||
}; |
||||
|
||||
const rootEvent = createIndexEventFromAST(parsed, ndk); |
||||
const tree = new PublicationTree(rootEvent, ndk); |
||||
|
||||
for (const section of parsed.sections) { |
||||
const contentEvent = createContentEventFromSection(section, ndk); |
||||
await tree.addEvent(contentEvent, rootEvent); |
||||
} |
||||
|
||||
return tree; |
||||
} |
||||
@ -0,0 +1,325 @@
@@ -0,0 +1,325 @@
|
||||
/** |
||||
* Factory for creating PublicationTree instances from AsciiDoc content |
||||
*
|
||||
* This integrates the AST parser with Michael's PublicationTree architecture, |
||||
* providing a clean bridge between AsciiDoc parsing and Nostr event publishing. |
||||
*/ |
||||
|
||||
import { PublicationTree } from "../data_structures/publication_tree.ts"; |
||||
import { SveltePublicationTree } from "../components/publications/svelte_publication_tree.svelte.ts"; |
||||
import { parseAsciiDocAST } from "./asciidoc_ast_parser.ts"; |
||||
import { NDKEvent } from "@nostr-dev-kit/ndk"; |
||||
import type NDK from "@nostr-dev-kit/ndk"; |
||||
import { getMimeTags } from "./mime.ts"; |
||||
|
||||
export interface PublicationTreeFactoryResult { |
||||
tree: PublicationTree; |
||||
svelteTree: SveltePublicationTree; |
||||
indexEvent: NDKEvent | null; |
||||
contentEvents: NDKEvent[]; |
||||
metadata: { |
||||
title: string; |
||||
totalSections: number; |
||||
contentType: 'article' | 'scattered-notes' | 'none'; |
||||
attributes: Record<string, string>; |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Create a PublicationTree from AsciiDoc content using AST parsing |
||||
* This is the main integration point between AST parsing and PublicationTree |
||||
*/ |
||||
export async function createPublicationTreeFromContent( |
||||
content: string, |
||||
ndk: NDK, |
||||
parseLevel: number = 2 |
||||
): Promise<PublicationTreeFactoryResult> { |
||||
|
||||
// For preview purposes, we can work without authentication
|
||||
// Authentication is only required for actual publishing
|
||||
const hasActiveUser = !!ndk.activeUser; |
||||
|
||||
// Parse content using AST
|
||||
const parsed = parseAsciiDocAST(content, parseLevel); |
||||
|
||||
// Determine content type
|
||||
const contentType = detectContentType(parsed); |
||||
|
||||
let tree: PublicationTree; |
||||
let indexEvent: NDKEvent | null = null; |
||||
const contentEvents: NDKEvent[] = []; |
||||
|
||||
if (contentType === 'article' && parsed.title) { |
||||
// Create hierarchical structure: 30040 index + 30041 content events
|
||||
indexEvent = createIndexEvent(parsed, ndk); |
||||
tree = new PublicationTree(indexEvent, ndk); |
||||
|
||||
// Add content events to tree
|
||||
for (const section of parsed.sections) { |
||||
const contentEvent = createContentEvent(section, parsed, ndk); |
||||
await tree.addEvent(contentEvent, indexEvent); |
||||
contentEvents.push(contentEvent); |
||||
} |
||||
|
||||
} else if (contentType === 'scattered-notes') { |
||||
// Create flat structure: only 30041 events
|
||||
if (parsed.sections.length === 0) { |
||||
throw new Error("No sections found for scattered notes"); |
||||
} |
||||
|
||||
// Use first section as root for tree structure
|
||||
const firstSection = parsed.sections[0]; |
||||
const rootEvent = createContentEvent(firstSection, parsed, ndk); |
||||
tree = new PublicationTree(rootEvent, ndk); |
||||
contentEvents.push(rootEvent); |
||||
|
||||
// Add remaining sections
|
||||
for (let i = 1; i < parsed.sections.length; i++) { |
||||
const contentEvent = createContentEvent(parsed.sections[i], parsed, ndk); |
||||
await tree.addEvent(contentEvent, rootEvent); |
||||
contentEvents.push(contentEvent); |
||||
} |
||||
|
||||
} else { |
||||
throw new Error("No valid content found to create publication tree"); |
||||
} |
||||
|
||||
// Create reactive Svelte wrapper
|
||||
const svelteTree = new SveltePublicationTree( |
||||
indexEvent || contentEvents[0],
|
||||
ndk |
||||
); |
||||
|
||||
return { |
||||
tree, |
||||
svelteTree, |
||||
indexEvent, |
||||
contentEvents, |
||||
metadata: { |
||||
title: parsed.title, |
||||
totalSections: parsed.sections.length, |
||||
contentType, |
||||
attributes: parsed.attributes |
||||
} |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Create a 30040 index event from parsed document |
||||
*/ |
||||
function createIndexEvent(parsed: any, ndk: NDK): NDKEvent { |
||||
const event = new NDKEvent(ndk); |
||||
event.kind = 30040; |
||||
event.created_at = Math.floor(Date.now() / 1000); |
||||
// Use placeholder pubkey for preview if no active user
|
||||
event.pubkey = ndk.activeUser?.pubkey || 'preview-placeholder-pubkey'; |
||||
|
||||
// Generate d-tag from title
|
||||
const dTag = generateDTag(parsed.title); |
||||
const [mTag, MTag] = getMimeTags(30040); |
||||
|
||||
const tags: string[][] = [ |
||||
["d", dTag], |
||||
mTag, |
||||
MTag, |
||||
["title", parsed.title] |
||||
]; |
||||
|
||||
// Add document attributes as tags
|
||||
addDocumentAttributesToTags(tags, parsed.attributes, event.pubkey); |
||||
|
||||
// Add a-tags for each section (30041 references)
|
||||
parsed.sections.forEach((section: any) => { |
||||
const sectionDTag = generateDTag(section.title); |
||||
tags.push(["a", `30041:${event.pubkey}:${sectionDTag}`]); |
||||
}); |
||||
|
||||
event.tags = tags; |
||||
event.content = parsed.content || generateIndexContent(parsed); |
||||
|
||||
return event; |
||||
} |
||||
|
||||
/** |
||||
* Create a 30041 content event from parsed section |
||||
*/ |
||||
function createContentEvent(section: any, documentParsed: any, ndk: NDK): NDKEvent { |
||||
const event = new NDKEvent(ndk); |
||||
event.kind = 30041; |
||||
event.created_at = Math.floor(Date.now() / 1000); |
||||
|
||||
// Use placeholder pubkey for preview if no active user
|
||||
event.pubkey = ndk.activeUser?.pubkey || 'preview-placeholder-pubkey'; |
||||
|
||||
const dTag = generateDTag(section.title); |
||||
const [mTag, MTag] = getMimeTags(30041); |
||||
|
||||
const tags: string[][] = [ |
||||
["d", dTag], |
||||
mTag, |
||||
MTag, |
||||
["title", section.title] |
||||
]; |
||||
|
||||
// Add section-specific attributes
|
||||
addSectionAttributesToTags(tags, section.attributes); |
||||
|
||||
// Add document-level attributes that should be inherited
|
||||
inheritDocumentAttributes(tags, documentParsed.attributes); |
||||
|
||||
event.tags = tags; |
||||
event.content = section.content || ''; |
||||
|
||||
return event; |
||||
} |
||||
|
||||
/** |
||||
* Detect content type based on parsed structure |
||||
*/ |
||||
function detectContentType(parsed: any): 'article' | 'scattered-notes' | 'none' { |
||||
const hasDocTitle = !!parsed.title; |
||||
const hasSections = parsed.sections.length > 0; |
||||
|
||||
// Check if the "title" is actually just the first section title
|
||||
// This happens when AsciiDoc starts with == instead of =
|
||||
const titleMatchesFirstSection = parsed.sections.length > 0 &&
|
||||
parsed.title === parsed.sections[0].title; |
||||
|
||||
if (hasDocTitle && hasSections && !titleMatchesFirstSection) { |
||||
return 'article'; |
||||
} else if (hasSections) { |
||||
return 'scattered-notes'; |
||||
} |
||||
|
||||
return 'none'; |
||||
} |
||||
|
||||
/** |
||||
* Generate deterministic d-tag from title |
||||
*/ |
||||
function generateDTag(title: string): string { |
||||
return title |
||||
.toLowerCase() |
||||
.replace(/[^\p{L}\p{N}]/gu, "-") |
||||
.replace(/-+/g, "-") |
||||
.replace(/^-|-$/g, "") || "untitled"; |
||||
} |
||||
|
||||
/** |
||||
* Add document attributes as Nostr tags |
||||
*/ |
||||
function addDocumentAttributesToTags(tags: string[][], attributes: Record<string, string>, pubkey: string) { |
||||
// Standard metadata
|
||||
if (attributes.author) tags.push(["author", attributes.author]); |
||||
if (attributes.version) tags.push(["version", attributes.version]); |
||||
if (attributes.published) tags.push(["published", attributes.published]); |
||||
if (attributes.language) tags.push(["language", attributes.language]); |
||||
if (attributes.image) tags.push(["image", attributes.image]); |
||||
if (attributes.description) tags.push(["summary", attributes.description]); |
||||
|
||||
// Tags
|
||||
if (attributes.tags) { |
||||
attributes.tags.split(',').forEach(tag =>
|
||||
tags.push(["t", tag.trim()]) |
||||
); |
||||
} |
||||
|
||||
// Add pubkey reference
|
||||
tags.push(["p", pubkey]); |
||||
|
||||
// Custom attributes (filtered)
|
||||
addCustomAttributes(tags, attributes); |
||||
} |
||||
|
||||
/** |
||||
* Add section-specific attributes as tags |
||||
*/ |
||||
function addSectionAttributesToTags(tags: string[][], attributes: Record<string, string>) { |
||||
addCustomAttributes(tags, attributes); |
||||
} |
||||
|
||||
/** |
||||
* Inherit relevant document attributes for content events |
||||
*/ |
||||
function inheritDocumentAttributes(tags: string[][], documentAttributes: Record<string, string>) { |
||||
// Inherit selected document attributes
|
||||
if (documentAttributes.language) tags.push(["language", documentAttributes.language]); |
||||
if (documentAttributes.type) tags.push(["type", documentAttributes.type]); |
||||
} |
||||
|
||||
/** |
||||
* Add custom attributes, filtering out system ones |
||||
*/ |
||||
function addCustomAttributes(tags: string[][], attributes: Record<string, string>) { |
||||
const systemAttributes = [ |
||||
'attribute-undefined', 'attribute-missing', 'appendix-caption', 'appendix-refsig', |
||||
'caution-caption', 'chapter-refsig', 'example-caption', 'figure-caption', |
||||
'important-caption', 'last-update-label', 'manname-title', 'note-caption', |
||||
'part-refsig', 'preface-title', 'section-refsig', 'table-caption', |
||||
'tip-caption', 'toc-title', 'untitled-label', 'version-label', 'warning-caption', |
||||
'asciidoctor', 'asciidoctor-version', 'safe-mode-name', 'backend', 'doctype', |
||||
'basebackend', 'filetype', 'outfilesuffix', 'stylesdir', 'iconsdir', |
||||
'localdate', 'localyear', 'localtime', 'localdatetime', 'docdate', |
||||
'docyear', 'doctime', 'docdatetime', 'doctitle', 'embedded', 'notitle', |
||||
// Already handled above
|
||||
'author', 'version', 'published', 'language', 'image', 'description', 'tags', 'title', 'type' |
||||
]; |
||||
|
||||
Object.entries(attributes).forEach(([key, value]) => { |
||||
if (!systemAttributes.includes(key) && value && typeof value === 'string') { |
||||
tags.push([key, value]); |
||||
} |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Generate default index content if none provided |
||||
*/ |
||||
function generateIndexContent(parsed: any): string { |
||||
return `# ${parsed.title} |
||||
|
||||
${parsed.sections.length} sections available: |
||||
|
||||
${parsed.sections.map((section: any, i: number) =>
|
||||
`${i + 1}. ${section.title}` |
||||
).join('\n')}`;
|
||||
} |
||||
|
||||
/** |
||||
* Export events from PublicationTree for publishing |
||||
* This provides compatibility with the current publishing workflow |
||||
*/ |
||||
export async function exportEventsFromTree(result: PublicationTreeFactoryResult) { |
||||
const events: any[] = []; |
||||
|
||||
// Add index event if it exists
|
||||
if (result.indexEvent) { |
||||
events.push(eventToPublishableObject(result.indexEvent)); |
||||
} |
||||
|
||||
// Add content events
|
||||
result.contentEvents.forEach(event => { |
||||
events.push(eventToPublishableObject(event)); |
||||
}); |
||||
|
||||
return { |
||||
indexEvent: result.indexEvent ? eventToPublishableObject(result.indexEvent) : undefined, |
||||
contentEvents: result.contentEvents.map(eventToPublishableObject), |
||||
tree: result.tree |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Convert NDKEvent to publishable object format |
||||
*/ |
||||
function eventToPublishableObject(event: NDKEvent) { |
||||
return { |
||||
kind: event.kind, |
||||
content: event.content, |
||||
tags: event.tags, |
||||
created_at: event.created_at, |
||||
pubkey: event.pubkey, |
||||
id: event.id, |
||||
title: event.tags.find(t => t[0] === 'title')?.[1] || 'Untitled' |
||||
}; |
||||
} |
||||
Loading…
Reference in new issue