diff --git a/assets/styles/03-components/card.css b/assets/styles/03-components/card.css
index 218babd..ce746d7 100644
--- a/assets/styles/03-components/card.css
+++ b/assets/styles/03-components/card.css
@@ -60,6 +60,13 @@ h2.card-title {
object-fit: cover;
}
+@media screen and (min-width: 1200px) {
+ .featured-list .card-header img {
+ max-height: initial !important;
+ aspect-ratio: 1.4;
+ }
+}
+
.card.comment {
display: flex;
flex-direction: column;
diff --git a/src/Command/NzineSortArticlesCommand.php b/src/Command/NzineSortArticlesCommand.php
new file mode 100644
index 0000000..da60bf1
--- /dev/null
+++ b/src/Command/NzineSortArticlesCommand.php
@@ -0,0 +1,268 @@
+nzineRepository->findOneBy([]);
+ if (!$nzine) {
+ $io->error('No NZine entity found.');
+ return Command::FAILURE;
+ }
+
+ /** @var NzineBot $bot */
+ $bot = $nzine->getNzineBot();
+ $bot->setEncryptionService($this->encryptionService);
+
+ $key = new Key();
+ $signer = new Sign();
+ $publicKey = strtolower($key->getPublicKey($bot->getNsec())); // hex
+
+ /** @var Article[] $articles */
+ $articles = $this->articleRepository->findBy(['pubkey' => $publicKey]);
+ $io->writeln('Articles for bot: ' . count($articles));
+
+ /** @var DbEvent[] $indexes */
+ $indexes = $this->em->getRepository(DbEvent::class)->findBy([
+ 'pubkey' => $publicKey,
+ 'kind' => KindsEnum::PUBLICATION_INDEX,
+ ]);
+ $io->writeln('Found ' . count($indexes) . ' existing indexes for bot ' . $publicKey);
+
+ if (!$indexes) {
+ $io->warning('No existing publication indexes found; nothing to update.');
+ return Command::SUCCESS;
+ }
+
+ // newest index per d-tag (slug)
+ $newestIndexBySlug = [];
+ foreach ($indexes as $idx) {
+ $d = $this->firstTagValue($idx->getTags() ?? [], 'd');
+ if ($d === null) continue;
+ if (!isset($newestIndexBySlug[$d]) || $idx->getCreatedAt() > $newestIndexBySlug[$d]->getCreatedAt()) {
+ $newestIndexBySlug[$d] = $idx;
+ }
+ }
+
+ $mainCategories = $nzine->getMainCategories() ?? [];
+ $totalUpdated = 0;
+
+ foreach ($mainCategories as $category) {
+ $slug = (string)($category['slug'] ?? '');
+ if ($slug === '') continue;
+
+ $index = $newestIndexBySlug[$slug] ?? null;
+ if (!$index) {
+ $io->writeln(" - Skip category '{$slug}': no index found for this slug.");
+ continue;
+ }
+
+ $tags = $index->getTags() ?? [];
+
+ // topics tracked by this index (t-tags)
+ $trackedTopics = array_values(array_unique(array_filter(array_map(
+ fn($t) => $this->normTag($t),
+ $this->allTagValues($tags, 't')
+ ))));
+ if (!$trackedTopics) {
+ $io->writeln(" - Index d='{$slug}': no tracked 't' tags, skipping.");
+ continue;
+ }
+
+ // existing a-tags for dedupe
+ $existingA = [];
+ foreach ($tags as $t) {
+ if (($t[0] ?? null) === 'a' && isset($t[1])) {
+ $existingA[strtolower($t[1])] = true;
+ }
+ }
+
+ $added = 0;
+
+ foreach ($articles as $article) {
+ if (strtolower($article->getPubkey()) !== $publicKey) continue;
+
+ $slugArticle = (string)$article->getSlug();
+ if ($slugArticle === '') continue;
+
+ $articleTopics = $article->getTopics() ?? [];
+ if (!$articleTopics) continue;
+
+ if (!$this->intersects($articleTopics, $trackedTopics)) continue;
+
+ $coord = sprintf('%s:%s:%s', KindsEnum::LONGFORM->value, $publicKey, $slugArticle);
+ $coordKey = strtolower($coord);
+ if (!isset($existingA[$coordKey])) {
+ $tags[] = ['a', $coord];
+ $existingA[$coordKey] = true;
+ $added++;
+ }
+ }
+
+ if ($added > 0) {
+ $tags = $this->sortedATagsLast($tags);
+ $index->setTags($tags);
+
+ // ---- SIGN USING SWENTEL EVENT ----
+ $wire = $this->toWireEvent($index, $publicKey);
+ $wire->setTags($tags);
+ $signer->signEvent($wire, $bot->getNsec());
+ $this->applySignedWireToEntity($wire, $index);
+ // -----------------------------------
+
+ $this->em->persist($index);
+ $io->writeln(" + Updated index d='{$slug}': added {$added} article(s).");
+ $totalUpdated++;
+ } else {
+ $io->writeln(" - Index d='{$slug}': no new matches.");
+ }
+ }
+
+ if ($totalUpdated > 0) {
+ $this->em->flush();
+ }
+
+ $io->success("Done. Updated {$totalUpdated} index(es).");
+ return Command::SUCCESS;
+ }
+
+ private function firstTagValue(array $tags, string $name): ?string
+ {
+ foreach ($tags as $t) {
+ if (($t[0] ?? null) === $name && isset($t[1])) {
+ return (string)$t[1];
+ }
+ }
+ return null;
+ }
+
+ private function allTagValues(array $tags, string $name): array
+ {
+ $out = [];
+ foreach ($tags as $t) {
+ if (($t[0] ?? null) === $name && isset($t[1])) {
+ $out[] = (string)$t[1];
+ }
+ }
+ return $out;
+ }
+
+ private function normTag(?string $t): string
+ {
+ $t = trim((string)$t);
+ if ($t !== '' && $t[0] === '#') $t = substr($t, 1);
+ return mb_strtolower($t);
+ }
+
+ private function intersects(array $a, array $b): bool
+ {
+ if (!$a || !$b) return false;
+ $set = array_fill_keys($b, true);
+ foreach ($a as $x) if (isset($set[$x])) return true;
+ return false;
+ }
+
+ private function sortedATagsLast(array $tags): array
+ {
+ $aTags = [];
+ $other = [];
+ foreach ($tags as $t) {
+ if (($t[0] ?? null) === 'a' && isset($t[1])) $aTags[] = $t;
+ else $other[] = $t;
+ }
+ usort($aTags, fn($x, $y) => strcmp(strtolower($x[1]), strtolower($y[1])));
+ return array_merge($other, $aTags);
+ }
+
+ /**
+ * Build a swentel wire event from your DB entity so we can sign it.
+ */
+ private function toWireEvent(DbEvent $e, string $pubkey): WireEvent
+ {
+ $w = new WireEvent();
+ $w->setKind($e->getKind());
+ $createdAt = $e->getCreatedAt();
+ // accept int or DateTimeInterface
+ if ($createdAt instanceof \DateTimeInterface) {
+ $w->setCreatedAt($createdAt->getTimestamp());
+ } else {
+ $w->setCreatedAt((int)$createdAt ?: time());
+ }
+ $w->setContent((string)($e->getContent() ?? ''));
+ $w->setTags($e->getTags() ?? []);
+ $w->setPublicKey($pubkey); // ensure pubkey is set for id computation
+ return $w;
+ }
+
+ /**
+ * Copy signature/id (and any normalized fields) back to your entity.
+ */
+ private function applySignedWireToEntity(WireEvent $w, DbEvent $e): void
+ {
+ if (method_exists($e, 'setId') && $w->getId()) {
+ $e->setId($w->getId());
+ }
+ if (method_exists($e, 'setSig') && $w->getSignature()) {
+ $e->setSig($w->getSignature());
+ }
+ if (method_exists($e, 'setPubkey') && $w->getPublicKey()) {
+ $e->setPubkey($w->getPublicKey());
+ }
+ // keep tags/content in sync (in case swentel normalized)
+ if (method_exists($e, 'setTags')) {
+ $e->setTags($w->getTags());
+ }
+ if (method_exists($e, 'setContent')) {
+ $e->setContent($w->getContent());
+ }
+ if (method_exists($e, 'setCreatedAt') && is_int($w->getCreatedAt())) {
+ // optional: keep your entity’s createdAt as int or DateTime, depending on your schema
+ try {
+ $e->setCreatedAt($w->getCreatedAt());
+ } catch (\TypeError $t) {
+ // if your setter expects DateTimeImmutable:
+ if ($w->getCreatedAt()) {
+ $e->setCreatedAt((new \DateTimeImmutable())->setTimestamp($w->getCreatedAt())->getTimestamp());
+ }
+ }
+ }
+ // also ensure kind stays set
+ if (method_exists($e, 'setKind')) {
+ $e->setKind($w->getKind());
+ }
+ }
+}
diff --git a/src/Command/RssFetchCommand.php b/src/Command/RssFetchCommand.php
index f998aa7..2e7dfe3 100644
--- a/src/Command/RssFetchCommand.php
+++ b/src/Command/RssFetchCommand.php
@@ -2,544 +2,385 @@
namespace App\Command;
+use App\Entity\Article;
+use App\Entity\NzineBot;
use App\Factory\ArticleFactory;
-use App\Repository\ArticleRepository;
use App\Repository\NzineRepository;
use App\Service\EncryptionService;
use App\Service\NostrClient;
-use App\Service\NzineCategoryIndexService;
use App\Service\RssFeedService;
-use App\Service\RssToNostrConverter;
-use App\Service\TagMatchingService;
use Doctrine\ORM\EntityManagerInterface;
-use Psr\Log\LoggerInterface;
+use League\HTMLToMarkdown\HtmlConverter;
+use swentel\nostr\Event\Event;
+use swentel\nostr\Key\Key;
+use swentel\nostr\Sign\Sign;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\String\Slugger\AsciiSlugger;
#[AsCommand(
name: 'nzine:rss:fetch',
- description: 'Fetch RSS feeds and publish as Nostr events for configured nzines',
+ description: 'Fetch RSS feeds and save new articles for configured nzines',
)]
class RssFetchCommand extends Command
{
- private SymfonyStyle $io;
-
public function __construct(
- private readonly NzineRepository $nzineRepository,
- private readonly ArticleRepository $articleRepository,
- private readonly RssFeedService $rssFeedService,
- private readonly TagMatchingService $tagMatchingService,
- private readonly RssToNostrConverter $rssToNostrConverter,
- private readonly ArticleFactory $articleFactory,
- private readonly NostrClient $nostrClient,
+ private readonly NzineRepository $nzineRepository,
+ private readonly ArticleFactory $factory,
+ private readonly RssFeedService $rssFeedService,
private readonly EntityManagerInterface $entityManager,
- private readonly EncryptionService $encryptionService,
- private readonly LoggerInterface $logger,
- private readonly NzineCategoryIndexService $categoryIndexService
+ private readonly NostrClient $nostrClient,
+ private readonly EncryptionService $encryptionService
) {
parent::__construct();
}
- protected function configure(): void
- {
- $this
- ->addOption('nzine-id', null, InputOption::VALUE_OPTIONAL, 'Process only this specific nzine ID')
- ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Test without actually publishing events')
- ->addOption('limit', null, InputOption::VALUE_OPTIONAL, 'Limit number of items to process per feed', 50);
- }
-
protected function execute(InputInterface $input, OutputInterface $output): int
{
- $this->io = new SymfonyStyle($input, $output);
+ $io = new SymfonyStyle($input, $output);
+ $slugger = new AsciiSlugger();
- $nzineId = $input->getOption('nzine-id');
- $isDryRun = $input->getOption('dry-run');
- $limit = (int) $input->getOption('limit');
+ $nzines = $this->nzineRepository->findAll();
+ foreach ($nzines as $nzine) {
+ if (!$nzine->getFeedUrl()) {
+ continue;
+ }
- $this->io->title('RSS Feed to Nostr Aggregator');
+ /** @var NzineBot $bot */
+ $bot = $nzine->getNzineBot();
+ $bot->setEncryptionService($this->encryptionService);
- if ($isDryRun) {
- $this->io->warning('Running in DRY-RUN mode - no events will be published');
- }
+ $key = new Key();
+ $npub = $key->getPublicKey($bot->getNsec());
+ $articles = $this->entityManager->getRepository(Article::class)->findBy(['pubkey' => $npub]);
+ $io->writeln('Found ' . count($articles) . ' existing articles for bot ' . $npub);
- // Get nzines to process
- $nzines = $nzineId
- ? [$this->nzineRepository->findRssNzineById((int) $nzineId)]
- : $this->nzineRepository->findActiveRssNzines();
+ $io->section('Fetching RSS for: ' . $nzine->getFeedUrl());
- $nzines = array_filter($nzines); // Remove nulls
+ try {
+ $feed = $this->rssFeedService->fetchFeed($nzine->getFeedUrl());
+ } catch (\Throwable $e) {
+ $io->warning('Failed to fetch ' . $nzine->getFeedUrl() . ': ' . $e->getMessage());
+ continue;
+ }
- if (empty($nzines)) {
- $this->io->warning('No RSS-enabled nzines found');
- return Command::SUCCESS;
- }
+ foreach ($feed['items'] as $item) {
+ try {
+ $event = new Event();
+ $event->setKind(30023); // NIP-23 Long-form content
+
+ // created_at — use parsed pubDate (timestamp int) or now
+ $createdAt = isset($item['pubDate']) && is_numeric($item['pubDate'])
+ ? (int)$item['pubDate']
+ : time();
+ $event->setCreatedAt($createdAt);
+
+ // slug (NIP-33 'd' tag) — stable per source item
+ $base = trim(($nzine->getSlug() ?? 'nzine') . '-' . ($item['title'] ?? ''));
+ $slug = (string) $slugger->slug($base)->lower();
+
+ // HTML → Markdown
+ $raw = trim($item['content'] ?? '') ?: trim($item['description'] ?? '');
+ $rawHtml = $this->normalizeWeirdHtml($raw);
+ $cleanHtml = $this->sanitizeHtml($rawHtml);
+ $markdown = $this->htmlToMarkdown($cleanHtml);
+ $event->setContent($markdown);
+
+ // Tags
+ $tags = [
+ ['title', $this->safeStr($item['title'] ?? '')],
+ ['d', $slug],
+ ['source', $this->safeStr($item['link'] ?? '')],
+ ];
+
+ // summary (short description)
+ $summary = $this->ellipsis($this->plainText($item['description'] ?? ''), 280);
+ if ($summary !== '') {
+ $tags[] = ['summary', $summary];
+ }
- $this->io->info(sprintf('Processing %d nzine(s)', count($nzines)));
+ // image
+ if (!empty($item['image'])) {
+ $tags[] = ['image', $this->safeStr($item['image'])];
+ } else {
+ // try to sniff first from content if media tag was missing
+ if (preg_match('~
]+src="([^"]+)"~i', $rawHtml, $m)) {
+ $tags[] = ['image', $m[1]];
+ }
+ }
- $totalStats = [
- 'nzines_processed' => 0,
- 'items_fetched' => 0,
- 'items_matched' => 0,
- 'items_skipped_duplicate' => 0,
- 'items_skipped_unmatched' => 0,
- 'events_created' => 0,
- 'events_updated' => 0,
- 'errors' => 0,
- ];
+ // categories → "t" tags
+ if (!empty($item['categories']) && is_array($item['categories'])) {
+ foreach ($item['categories'] as $category) {
+ $cat = trim((string)$category);
+ if ($cat !== '') {
+ $event->addTag(['t', $cat]);
+ }
+ }
+ }
- foreach ($nzines as $nzine) {
- try {
- $stats = $this->processNzine($nzine, $isDryRun, $limit);
+ $event->setTags($tags);
+
+ // Sign event
+ $signer = new Sign();
+ $signer->signEvent($event, $bot->getNsec());
+
+ // Publish (add/adjust relays as you like)
+ try {
+ $this->nostrClient->publishEvent($event, [
+ 'wss://purplepag.es',
+ 'wss://relay.damus.io',
+ 'wss://nos.lol',
+ ]);
+ $io->writeln('Published long-form event: ' . ($item['title'] ?? '(no title)'));
+ } catch (\Throwable $e) {
+ $io->warning('Publish failed: ' . $e->getMessage());
+ }
- // Aggregate stats
- foreach ($stats as $key => $value) {
- $totalStats[$key] = ($totalStats[$key] ?? 0) + $value;
+ // Persist locally
+ $article = $this->factory->createFromLongFormContentEvent((object)$event->toArray());
+ $this->entityManager->persist($article);
+
+ } catch (\Throwable $e) {
+ // keep going on item errors
+ $io->warning('Item failed: ' . ($item['title'] ?? '(no title)') . ' — ' . $e->getMessage());
}
+ }
- $totalStats['nzines_processed']++;
- } catch (\Exception $e) {
- $this->io->error(sprintf(
- 'Error processing nzine #%d: %s',
- $nzine->getId(),
- $e->getMessage()
- ));
- $this->logger->error('Nzine processing error', [
- 'nzine_id' => $nzine->getId(),
- 'error' => $e->getMessage(),
- 'trace' => $e->getTraceAsString(),
- ]);
- $totalStats['errors']++;
+ $this->entityManager->flush();
+ $io->success('RSS fetch complete for: ' . $nzine->getFeedUrl());
+
+ // --- Update bot profile (kind 0) using feed metadata ---
+ $feedMeta = $feed['feed'] ?? null;
+ if ($feedMeta) {
+ $profile = [
+ 'name' => $feedMeta['title'] ?? $nzine->getTitle(),
+ 'about' => $feedMeta['description'] ?? '',
+ 'picture' => $feedMeta['image'] ?? null,
+ 'website' => $feedMeta['link'] ?? null,
+ ];
+ $p = new Event();
+ $p->setKind(0);
+ $p->setCreatedAt(time());
+ $p->setContent(json_encode($profile, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
+ $signer = new Sign();
+ $signer->signEvent($p, $bot->getNsec());
+ try {
+ $this->nostrClient->publishEvent($p, ['wss://purplepag.es']);
+ $io->success('Published bot profile (kind 0) with feed metadata');
+ } catch (\Throwable $e) {
+ $io->warning('Failed to publish bot profile event: ' . $e->getMessage());
+ }
}
}
- // Display final statistics
- $this->io->success('RSS feed processing completed');
- $this->io->table(
- ['Metric', 'Count'],
- [
- ['Nzines processed', $totalStats['nzines_processed']],
- ['Items fetched', $totalStats['items_fetched']],
- ['Items matched', $totalStats['items_matched']],
- ['Events created', $totalStats['events_created']],
- ['Events updated', $totalStats['events_updated']],
- ['Duplicates skipped', $totalStats['items_skipped_duplicate']],
- ['Unmatched skipped', $totalStats['items_skipped_unmatched']],
- ['Errors', $totalStats['errors']],
- ]
- );
-
- return $totalStats['errors'] > 0 ? Command::FAILURE : Command::SUCCESS;
+ return Command::SUCCESS;
}
- /**
- * Process a single nzine's RSS feed
- */
- private function processNzine($nzine, bool $isDryRun, int $limit): array
+ /** -------- Helpers: HTML prep + converter + small utils -------- */
+
+ private function normalizeWeirdHtml(string $html): string
{
- $stats = [
- 'items_fetched' => 0,
- 'items_matched' => 0,
- 'items_skipped_duplicate' => 0,
- 'items_skipped_unmatched' => 0,
- 'events_created' => 0,
- 'events_updated' => 0,
- ];
-
- $this->io->section(sprintf('Processing Nzine #%d: %s', $nzine->getId(), $nzine->getSlug()));
-
- $feedUrl = $nzine->getFeedUrl();
- if (empty($feedUrl)) {
- $this->io->warning('No feed URL configured');
- return $stats;
- }
+ // 1) Unwrap Ghost "HTML cards": keep only the
⚡ Tips: ' . htmlspecialchars($m[1]) . '
', + $html + ); + // Ghost/Koenig wrappers: keep useful inner content + $html = preg_replace('~$1', $html); + // YouTube iframes → links + $html = preg_replace_callback( + '~~i', + fn($m) => '', + $html + ); + + // 2) Try to pretty up malformed markup via Tidy (if available) + if (function_exists('tidy_parse_string')) { try { - $categoryIndices = $this->categoryIndexService->ensureCategoryIndices($nzine); - $this->io->text(sprintf('Category indices ready: %d', count($categoryIndices))); - } catch (\Exception $e) { - $this->io->warning(sprintf('Could not create category indices: %s', $e->getMessage())); - $this->logger->warning('Category index creation failed', [ - 'nzine_id' => $nzine->getId(), - 'error' => $e->getMessage(), - ]); - // Continue processing even if category indices fail + $tidy = tidy_parse_string($html, [ + 'clean' => true, + 'output-xhtml' => true, + 'show-body-only' => false, + 'wrap' => 0, + 'drop-empty-paras' => true, + 'merge-divs' => true, + 'merge-spans' => true, + 'numeric-entities' => false, + 'quote-ampersand' => true, + ], 'utf8'); + $tidy->cleanRepair(); + $html = (string)$tidy; + } catch (\Throwable $e) { + // ignore tidy failures } } - // Process each feed item - $this->io->progressStart(count($feedItems)); - - foreach ($feedItems as $item) { - $this->io->progressAdvance(); - - try { - $result = $this->processRssItem($item, $nzine, $categories, $isDryRun, $categoryIndices); - - if ($result === 'created') { - $stats['events_created']++; - $stats['items_matched']++; - } elseif ($result === 'updated') { - $stats['events_updated']++; - $stats['items_matched']++; - } elseif ($result === 'duplicate') { - $stats['items_skipped_duplicate']++; - } elseif ($result === 'unmatched') { - $stats['items_skipped_unmatched']++; - } - } catch (\Exception $e) { - $this->io->error(sprintf( - 'Error processing RSS item "%s": %s', - $item['title'] ?? 'unknown', - $e->getMessage() - )); - $this->logger->error('Error processing RSS item', [ - 'nzine_id' => $nzine->getId(), - 'item_title' => $item['title'] ?? 'unknown', - 'error' => $e->getMessage(), - 'trace' => $e->getTraceAsString(), - ]); - } + // 3) DOM sanitize: remove junk, unwrap html/body/head, allowlist elements/attrs + $dom = new \DOMDocument('1.0', 'UTF-8'); + libxml_use_internal_errors(true); + $loaded = $dom->loadHTML( + // force UTF-8 meta so DOMDocument doesn't mangle + ''.$html, + LIBXML_NOWARNING | LIBXML_NOERROR + ); + libxml_clear_errors(); + if (!$loaded) { + // fallback: as-is minus tags we already stripped + return $html; } - $this->io->progressFinish(); + $xpath = new \DOMXPath($dom); - // Re-sign all category indices after articles have been added - if (!$isDryRun && !empty($categoryIndices)) { - $this->io->text('Re-signing category indices...'); - try { - $this->categoryIndexService->resignCategoryIndices($categoryIndices, $nzine); - $this->io->text(sprintf('✓ Re-signed %d category indices', count($categoryIndices))); - } catch (\Exception $e) { - $this->io->warning(sprintf('Failed to re-sign category indices: %s', $e->getMessage())); - $this->logger->error('Category index re-signing failed', [ - 'nzine_id' => $nzine->getId(), - 'error' => $e->getMessage(), - ]); + // Remove ,