From 97c20a13bb287081a08df70fd63323550da2365f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nu=C5=A1a=20Puk=C5=A1i=C4=8D?= Date: Tue, 7 Oct 2025 17:10:36 +0200 Subject: [PATCH] Magazine from RSS --- assets/styles/05-utilities/utilities.css | 1 + migrations/Version20251007000000.php | 34 ++ src/Command/RssFetchCommand.php | 507 ++++++++++++++++++ .../MagazineAdminController.php | 126 ++++- src/Controller/DefaultController.php | 28 +- src/Controller/MagazineWizardController.php | 36 +- src/Controller/NzineController.php | 385 ++++++++++--- src/Entity/Article.php | 6 + src/Entity/Event.php | 2 +- src/Entity/Nzine.php | 39 ++ src/Examples/RssNzineSetupExample.php | 159 ++++++ src/Form/MainCategoryType.php | 10 + src/Form/NzineBotType.php | 30 +- src/Repository/NzineRepository.php | 29 + src/Service/RssFeedService.php | 189 +++++++ src/Service/RssToNostrConverter.php | 166 ++++++ src/Service/TagMatchingService.php | 84 +++ src/Twig/Components/Header.php | 17 - .../Components/Molecules/CategoryLink.php | 24 +- .../Components/Organisms/FeaturedList.php | 32 +- templates/admin/magazines.html.twig | 4 +- templates/admin/magazines_orphaned.html.twig | 64 +++ .../Organisms/FeaturedList.html.twig | 4 +- .../ReadingListWorkflowStatus.html.twig | 2 +- templates/layout.html.twig | 6 +- templates/nzine/list.html.twig | 57 ++ templates/pages/nzine-editor.html.twig | 115 +++- tests/Service/TagMatchingServiceTest.php | 157 ++++++ 28 files changed, 2161 insertions(+), 152 deletions(-) create mode 100644 migrations/Version20251007000000.php create mode 100644 src/Command/RssFetchCommand.php create mode 100644 src/Examples/RssNzineSetupExample.php create mode 100644 src/Service/RssFeedService.php create mode 100644 src/Service/RssToNostrConverter.php create mode 100644 src/Service/TagMatchingService.php create mode 100644 templates/admin/magazines_orphaned.html.twig create mode 100644 templates/nzine/list.html.twig create mode 100644 tests/Service/TagMatchingServiceTest.php diff --git a/assets/styles/05-utilities/utilities.css b/assets/styles/05-utilities/utilities.css index 1cabdfa..9c0ecfc 100644 --- a/assets/styles/05-utilities/utilities.css +++ b/assets/styles/05-utilities/utilities.css @@ -43,6 +43,7 @@ /* Alerts */ .alert{position:relative;padding:.75rem 1rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem} +.alert-default{color:var(--color-text);background-color:var(--color-secondary-bg);border-color:var(--color-secondary-bg)} .alert-info{color:#055160;background-color:#e9f5ff;border-color:#b6e0fe} .alert-success{color:#0f5132;background-color:#edf7ed;border-color:#c6e6c6} .alert-danger{color:#842029;background-color:#f8d7da;border-color:#f5c2c7} diff --git a/migrations/Version20251007000000.php b/migrations/Version20251007000000.php new file mode 100644 index 0000000..2f776aa --- /dev/null +++ b/migrations/Version20251007000000.php @@ -0,0 +1,34 @@ +addSql('ALTER TABLE nzine ADD feed_url TEXT DEFAULT NULL'); + $this->addSql('ALTER TABLE nzine ADD last_fetched_at TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL'); + $this->addSql('ALTER TABLE nzine ADD feed_config JSON DEFAULT NULL'); + $this->addSql('COMMENT ON COLUMN nzine.last_fetched_at IS \'(DC2Type:datetime_immutable)\''); + } + + public function down(Schema $schema): void + { + $this->addSql('ALTER TABLE nzine DROP feed_url'); + $this->addSql('ALTER TABLE nzine DROP last_fetched_at'); + $this->addSql('ALTER TABLE nzine DROP feed_config'); + } +} diff --git a/src/Command/RssFetchCommand.php b/src/Command/RssFetchCommand.php new file mode 100644 index 0000000..bee215e --- /dev/null +++ b/src/Command/RssFetchCommand.php @@ -0,0 +1,507 @@ +addOption('nzine-id', null, InputOption::VALUE_OPTIONAL, 'Process only this specific nzine ID') + ->addOption('dry-run', null, InputOption::VALUE_NONE, 'Test without actually publishing events') + ->addOption('limit', null, InputOption::VALUE_OPTIONAL, 'Limit number of items to process per feed', 50); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->io = new SymfonyStyle($input, $output); + + $nzineId = $input->getOption('nzine-id'); + $isDryRun = $input->getOption('dry-run'); + $limit = (int) $input->getOption('limit'); + + $this->io->title('RSS Feed to Nostr Aggregator'); + + if ($isDryRun) { + $this->io->warning('Running in DRY-RUN mode - no events will be published'); + } + + // Get nzines to process + $nzines = $nzineId + ? [$this->nzineRepository->findRssNzineById((int) $nzineId)] + : $this->nzineRepository->findActiveRssNzines(); + + $nzines = array_filter($nzines); // Remove nulls + + if (empty($nzines)) { + $this->io->warning('No RSS-enabled nzines found'); + return Command::SUCCESS; + } + + $this->io->info(sprintf('Processing %d nzine(s)', count($nzines))); + + $totalStats = [ + 'nzines_processed' => 0, + 'items_fetched' => 0, + 'items_matched' => 0, + 'items_skipped_duplicate' => 0, + 'items_skipped_unmatched' => 0, + 'events_created' => 0, + 'events_updated' => 0, + 'errors' => 0, + ]; + + foreach ($nzines as $nzine) { + try { + $stats = $this->processNzine($nzine, $isDryRun, $limit); + + // Aggregate stats + foreach ($stats as $key => $value) { + $totalStats[$key] = ($totalStats[$key] ?? 0) + $value; + } + + $totalStats['nzines_processed']++; + } catch (\Exception $e) { + $this->io->error(sprintf( + 'Error processing nzine #%d: %s', + $nzine->getId(), + $e->getMessage() + )); + $this->logger->error('Nzine processing error', [ + 'nzine_id' => $nzine->getId(), + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + $totalStats['errors']++; + } + } + + // Display final statistics + $this->io->success('RSS feed processing completed'); + $this->io->table( + ['Metric', 'Count'], + [ + ['Nzines processed', $totalStats['nzines_processed']], + ['Items fetched', $totalStats['items_fetched']], + ['Items matched', $totalStats['items_matched']], + ['Events created', $totalStats['events_created']], + ['Events updated', $totalStats['events_updated']], + ['Duplicates skipped', $totalStats['items_skipped_duplicate']], + ['Unmatched skipped', $totalStats['items_skipped_unmatched']], + ['Errors', $totalStats['errors']], + ] + ); + + return $totalStats['errors'] > 0 ? Command::FAILURE : Command::SUCCESS; + } + + /** + * Process a single nzine's RSS feed + */ + private function processNzine($nzine, bool $isDryRun, int $limit): array + { + $stats = [ + 'items_fetched' => 0, + 'items_matched' => 0, + 'items_skipped_duplicate' => 0, + 'items_skipped_unmatched' => 0, + 'events_created' => 0, + 'events_updated' => 0, + ]; + + $this->io->section(sprintf('Processing Nzine #%d: %s', $nzine->getId(), $nzine->getSlug())); + + $feedUrl = $nzine->getFeedUrl(); + if (empty($feedUrl)) { + $this->io->warning('No feed URL configured'); + return $stats; + } + + // Fetch RSS feed + try { + $feedItems = $this->rssFeedService->fetchFeed($feedUrl); + $stats['items_fetched'] = count($feedItems); + + $this->io->text(sprintf('Fetched %d items from feed', count($feedItems))); + } catch (\Exception $e) { + $this->io->error(sprintf('Failed to fetch feed: %s', $e->getMessage())); + throw $e; + } + + // Limit items if specified + if ($limit > 0 && count($feedItems) > $limit) { + $feedItems = array_slice($feedItems, 0, $limit); + $this->io->text(sprintf('Limited to %d items', $limit)); + } + + // Get nzine categories + $categories = $nzine->getMainCategories(); + if (empty($categories)) { + $this->io->warning('No categories configured - skipping all items'); + $stats['items_skipped_unmatched'] = count($feedItems); + return $stats; + } + + // Ensure category index events exist in the database + $categoryIndices = []; + if (!$isDryRun) { + $this->io->text('Ensuring category index events exist...'); + try { + $categoryIndices = $this->categoryIndexService->ensureCategoryIndices($nzine); + $this->io->text(sprintf('Category indices ready: %d', count($categoryIndices))); + } catch (\Exception $e) { + $this->io->warning(sprintf('Could not create category indices: %s', $e->getMessage())); + $this->logger->warning('Category index creation failed', [ + 'nzine_id' => $nzine->getId(), + 'error' => $e->getMessage(), + ]); + // Continue processing even if category indices fail + } + } + + // Process each feed item + $this->io->progressStart(count($feedItems)); + + foreach ($feedItems as $item) { + $this->io->progressAdvance(); + + try { + $result = $this->processRssItem($item, $nzine, $categories, $isDryRun, $categoryIndices); + + if ($result === 'created') { + $stats['events_created']++; + $stats['items_matched']++; + } elseif ($result === 'updated') { + $stats['events_updated']++; + $stats['items_matched']++; + } elseif ($result === 'duplicate') { + $stats['items_skipped_duplicate']++; + } elseif ($result === 'unmatched') { + $stats['items_skipped_unmatched']++; + } + } catch (\Exception $e) { + $this->io->error(sprintf( + 'Error processing RSS item "%s": %s', + $item['title'] ?? 'unknown', + $e->getMessage() + )); + $this->logger->error('Error processing RSS item', [ + 'nzine_id' => $nzine->getId(), + 'item_title' => $item['title'] ?? 'unknown', + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + } + } + + $this->io->progressFinish(); + + // Re-sign all category indices after articles have been added + if (!$isDryRun && !empty($categoryIndices)) { + $this->io->text('Re-signing category indices...'); + try { + $this->categoryIndexService->resignCategoryIndices($categoryIndices, $nzine); + $this->io->text(sprintf('✓ Re-signed %d category indices', count($categoryIndices))); + } catch (\Exception $e) { + $this->io->warning(sprintf('Failed to re-sign category indices: %s', $e->getMessage())); + $this->logger->error('Category index re-signing failed', [ + 'nzine_id' => $nzine->getId(), + 'error' => $e->getMessage(), + ]); + } + } + + // Update last fetched timestamp + if (!$isDryRun) { + $nzine->setLastFetchedAt(new \DateTimeImmutable()); + $this->entityManager->flush(); + } + + $this->io->table( + ['Metric', 'Count'], + [ + ['Items fetched', $stats['items_fetched']], + ['Items matched', $stats['items_matched']], + ['Events created', $stats['events_created']], + ['Events updated', $stats['events_updated']], + ['Duplicates skipped', $stats['items_skipped_duplicate']], + ['Unmatched skipped', $stats['items_skipped_unmatched']], + ] + ); + + return $stats; + } + + /** + * Process a single RSS item + * + * @return string Result: 'created', 'duplicate', or 'unmatched' + */ + private function processRssItem(array $item, $nzine, array $categories, bool $isDryRun, array $categoryIndices): string + { + // Generate slug for duplicate detection + $slug = $this->rssToNostrConverter->generateSlugForItem($item); + + // Check if already exists + $existing = $this->articleRepository->findOneBy(['slug' => $slug]); + if ($existing) { + if ($isDryRun) { + $this->io->text(sprintf( + ' 🔄 Would update: "%s"', + $item['title'] ?? 'unknown' + )); + return 'updated'; + } + + $this->io->text(sprintf( + ' 🔄 Updating existing article: "%s"', + $item['title'] ?? 'unknown' + )); + $this->logger->debug('Found existing article - updating', [ + 'slug' => $slug, + 'title' => $item['title'], + ]); + + // Match to category for fresh data + $matchedCategory = $this->tagMatchingService->findMatchingCategory( + $item['categories'] ?? [], + $categories + ); + + // Convert to Nostr event to get fresh data with all processing applied + $nostrEvent = $this->rssToNostrConverter->convertToNostrEvent( + $item, + $matchedCategory, + $nzine + ); + + // Add original RSS categories as additional tags + if (!empty($item['categories'])) { + foreach ($item['categories'] as $rssCategory) { + $categorySlug = strtolower(trim($rssCategory)); + $tagExists = false; + + foreach ($nostrEvent->getTags() as $existingTag) { + if (is_array($existingTag) && $existingTag[0] === 't' && isset($existingTag[1]) && $existingTag[1] === $categorySlug) { + $tagExists = true; + break; + } + } + + if (!$tagExists) { + $nostrEvent->addTag(['t', $categorySlug]); + } + } + } + + // Convert to stdClass for processing + $eventObject = json_decode($nostrEvent->toJson()); + + // Update all fields from the fresh event data + $existing->setContent($eventObject->content); + $existing->setTitle($item['title'] ?? ''); + + // Set createdAt and publishedAt from RSS pubDate if available + if (isset($item['pubDate']) && $item['pubDate'] instanceof \DateTimeImmutable) { + $existing->setCreatedAt($item['pubDate']); + $existing->setPublishedAt($item['pubDate']); + } + + // Extract and set summary from tags (now with HTML stripped) + foreach ($eventObject->tags as $tag) { + if ($tag[0] === 'summary' && isset($tag[1])) { + $existing->setSummary($tag[1]); + break; + } + } + + // Clear existing topics and re-add from fresh data + $existing->clearTopics(); + foreach ($eventObject->tags as $tag) { + if ($tag[0] === 't' && isset($tag[1])) { + $existing->addTopic($tag[1]); + } + } + + $this->entityManager->persist($existing); + $this->entityManager->flush(); + + $this->logger->info('Article updated with fresh RSS data', [ + 'slug' => $slug, + 'title' => $item['title'], + ]); + + return 'updated'; + } + + // Match to category + $matchedCategory = $this->tagMatchingService->findMatchingCategory( + $item['categories'] ?? [], + $categories + ); + + if (!$matchedCategory) { + $this->io->text(sprintf( + ' ℹ No category match: "%s" [categories: %s] - importing as standalone', + $item['title'] ?? 'unknown', + implode(', ', $item['categories'] ?? ['none']) + )); + $this->logger->debug('No category match for item - importing as standalone', [ + 'title' => $item['title'], + 'categories' => $item['categories'] ?? [], + ]); + // Don't return - continue processing without a category + } + + if ($isDryRun) { + $categoryLabel = $matchedCategory + ? ($matchedCategory['name'] ?? $matchedCategory['title'] ?? $matchedCategory['slug'] ?? 'unknown') + : 'standalone'; + + $this->io->text(sprintf( + ' ✓ Would create: "%s" → %s', + $item['title'] ?? 'unknown', + $categoryLabel + )); + $this->logger->info('[DRY RUN] Would create event', [ + 'title' => $item['title'], + 'category' => $categoryLabel, + 'slug' => $slug, + ]); + return 'created'; + } + + // Convert to Nostr event (with or without category) + $nostrEvent = $this->rssToNostrConverter->convertToNostrEvent( + $item, + $matchedCategory, + $nzine + ); + + // Add original RSS categories as additional tags (topics) + // This ensures RSS feed categories are preserved even if they don't match nzine categories + if (!empty($item['categories'])) { + foreach ($item['categories'] as $rssCategory) { + // Add as 't' tag if not already present + $categorySlug = strtolower(trim($rssCategory)); + $tagExists = false; + + foreach ($nostrEvent->getTags() as $existingTag) { + if (is_array($existingTag) && $existingTag[0] === 't' && isset($existingTag[1]) && $existingTag[1] === $categorySlug) { + $tagExists = true; + break; + } + } + + if (!$tagExists) { + $nostrEvent->addTag(['t', $categorySlug]); + } + } + } + + // Convert Nostr Event to stdClass object for ArticleFactory + $eventObject = json_decode($nostrEvent->toJson()); + + // Create Article entity from the event object + $article = $this->articleFactory->createFromLongFormContentEvent($eventObject); + $this->entityManager->persist($article); + $this->entityManager->flush(); + + // Add article to category index if category matched + if ($matchedCategory && isset($matchedCategory['slug']) && !empty($categoryIndices)) { + $categorySlug = $matchedCategory['slug']; + if (isset($categoryIndices[$categorySlug])) { + $articleCoordinate = sprintf( + '%d:%s:%s', + $article->getKind()->value, + $article->getPubkey(), + $article->getSlug() + ); + + try { + $this->categoryIndexService->addArticleToCategoryIndex( + $categoryIndices[$categorySlug], + $articleCoordinate + ); + $this->entityManager->flush(); + } catch (\Exception $e) { + $this->logger->warning('Failed to add article to category index', [ + 'article_slug' => $article->getSlug(), + 'category_slug' => $categorySlug, + 'error' => $e->getMessage(), + ]); + } + } + } + + $categoryLabel = $matchedCategory + ? ($matchedCategory['name'] ?? $matchedCategory['title'] ?? $matchedCategory['slug'] ?? 'unknown') + : 'standalone'; + + $this->io->text(sprintf( + ' ✓ Created: "%s" → %s', + $item['title'] ?? 'unknown', + $categoryLabel + )); + + // Publish to relays (async/background in production) + try { + // TODO: Get configured relays from nzine or use default + // $this->nostrClient->publishEvent($nostrEvent, $relays); + $this->logger->info('Event created and saved', [ + 'event_id' => $nostrEvent->getId() ?? 'unknown', + 'title' => $item['title'], + 'category' => $categoryLabel, + ]); + } catch (\Exception $e) { + $this->logger->warning('Failed to publish to relays', [ + 'event_id' => $nostrEvent->getId() ?? 'unknown', + 'error' => $e->getMessage(), + ]); + // Continue even if relay publishing fails + } + + return 'created'; + } +} + diff --git a/src/Controller/Administration/MagazineAdminController.php b/src/Controller/Administration/MagazineAdminController.php index a7d6dcf..78a0c32 100644 --- a/src/Controller/Administration/MagazineAdminController.php +++ b/src/Controller/Administration/MagazineAdminController.php @@ -6,6 +6,7 @@ namespace App\Controller\Administration; use App\Entity\Article; use App\Entity\Event; +use App\Entity\Nzine; use App\Enum\KindsEnum; use Doctrine\ORM\EntityManagerInterface; use Redis as RedisClient; @@ -29,6 +30,74 @@ class MagazineAdminController extends AbstractController ]); } + #[Route('/admin/magazines/{npub}/delete', name: 'admin_magazine_delete', methods: ['POST'])] + #[IsGranted('ROLE_ADMIN')] + public function delete(string $npub, EntityManagerInterface $em): Response + { + try { + // Find and delete all events associated with this nzine (main index + category indices) + $events = $em->getRepository(Event::class)->findBy([ + 'pubkey' => $npub, + 'kind' => KindsEnum::PUBLICATION_INDEX->value + ]); + + $deletedCount = count($events); + + foreach ($events as $event) { + $em->remove($event); + } + + // Also delete the Nzine entity itself + $nzine = $em->getRepository(\App\Entity\Nzine::class)->findOneBy(['npub' => $npub]); + if ($nzine) { + $em->remove($nzine); + } + + $em->flush(); + + $this->addFlash('success', sprintf( + 'Deleted nzine and %d associated index events.', + $deletedCount + )); + } catch (\Exception $e) { + $this->addFlash('error', 'Failed to delete nzine: ' . $e->getMessage()); + } + + return $this->redirectToRoute('admin_magazines'); + } + + #[Route('/admin/magazines/orphaned', name: 'admin_magazines_orphaned')] + #[IsGranted('ROLE_ADMIN')] + public function listOrphaned(EntityManagerInterface $em): Response + { + + // Find nzines (entities) + $nzines = $em->getRepository(\App\Entity\Nzine::class)->findAll(); + $nzineNpubs = array_map(fn($n) => $n->getNpub(), $nzines); + + // Also find malformed nzines (have Nzine entity but no or broken indices) + $malformed = []; + foreach ($nzines as $nzine) { + $npub = $nzine->getNpub(); + $hasIndices = isset($indexesByPubkey[$npub]) && !empty($indexesByPubkey[$npub]); + + if (!$hasIndices || empty($nzine->getSlug())) { + $malformed[] = [ + 'nzine' => $nzine, + 'npub' => $npub, + 'slug' => $nzine->getSlug(), + 'state' => $nzine->getState(), + 'categories' => count($nzine->getMainCategories()), + 'indices' => $indexesByPubkey[$npub] ?? [], + ]; + } + } + + return $this->render('admin/magazines_orphaned.html.twig', [ + 'malformed' => $nzines, + ]); + } + private function getMagazinesFromDatabase(EntityManagerInterface $em, RedisClient $redis, CacheInterface $redisCache): array { // 1) Get magazine events directly from database using indexed queries @@ -321,9 +390,18 @@ class MagazineAdminController extends AbstractController // ignore set errors } + // Check for main magazine in database instead of Redis try { - $main = $redisCache->get('magazine-newsroom-magazine-by-newsroom', fn() => null); - if ($main && !in_array('newsroom-magazine-by-newsroom', $slugs, true)) { + $conn = $em->getConnection(); + $sql = "SELECT e.* FROM event e + WHERE e.tags::jsonb @> ?::jsonb + LIMIT 1"; + $result = $conn->executeQuery($sql, [ + json_encode([['d', 'newsroom-magazine-by-newsroom']]) + ]); + $mainEventData = $result->fetchAssociative(); + + if ($mainEventData !== false && !in_array('newsroom-magazine-by-newsroom', $slugs, true)) { $slugs[] = 'newsroom-magazine-by-newsroom'; } } catch (\Throwable) { @@ -331,9 +409,9 @@ class MagazineAdminController extends AbstractController } $magazines = []; - $parse = function($event): array { + $parse = function(array $tags): array { $title = null; $slug = null; $a = []; - foreach ((array) $event->getTags() as $tag) { + foreach ($tags as $tag) { if (!is_array($tag) || !isset($tag[0])) continue; if ($tag[0] === 'title' && isset($tag[1])) $title = $tag[1]; if ($tag[0] === 'd' && isset($tag[1])) $slug = $tag[1]; @@ -346,11 +424,27 @@ class MagazineAdminController extends AbstractController ]; }; + $conn = $em->getConnection(); + $sql = "SELECT e.* FROM event e + WHERE e.tags::jsonb @> ?::jsonb + LIMIT 1"; + foreach ($slugs as $slug) { - $event = $redisCache->get('magazine-' . $slug, fn() => null); - if (!$event || !method_exists($event, 'getTags')) continue; + // Query database for magazine event + try { + $result = $conn->executeQuery($sql, [ + json_encode([['d', $slug]]) + ]); + $eventData = $result->fetchAssociative(); + + if ($eventData === false) continue; + + $tags = json_decode($eventData['tags'], true); + } catch (\Throwable) { + continue; + } - $data = $parse($event); + $data = $parse($tags); $categories = []; foreach ($data['a'] as $coord) { @@ -359,10 +453,22 @@ class MagazineAdminController extends AbstractController if (count($parts) !== 3) continue; $catSlug = $parts[2]; - $catEvent = $redisCache->get('magazine-' . $catSlug, fn() => null); - if (!$catEvent || !method_exists($catEvent, 'getTags')) continue; - $catData = $parse($catEvent); + // Query database for category event + try { + $catResult = $conn->executeQuery($sql, [ + json_encode([['d', $catSlug]]) + ]); + $catEventData = $catResult->fetchAssociative(); + + if ($catEventData === false) continue; + + $catTags = json_decode($catEventData['tags'], true); + } catch (\Throwable) { + continue; + } + + $catData = $parse($catTags); $files = []; $repo = $em->getRepository(Article::class); diff --git a/src/Controller/DefaultController.php b/src/Controller/DefaultController.php index ba95c9c..e4851e8 100644 --- a/src/Controller/DefaultController.php +++ b/src/Controller/DefaultController.php @@ -75,23 +75,37 @@ class DefaultController extends AbstractController * @throws InvalidArgumentException */ #[Route('/mag/{mag}/cat/{slug}', name: 'magazine-category')] - public function magCategory($mag, $slug, CacheInterface $redisCache, + public function magCategory($mag, $slug, EntityManagerInterface $entityManager, RedisCacheService $redisCacheService, FinderInterface $finder, LoggerInterface $logger): Response { $magazine = $redisCacheService->getMagazineIndex($mag); - $catIndex = $redisCache->get('magazine-' . $slug, function (){ - throw new Exception('Not found'); - }); + // Query the database for the category event by slug using native SQL + $sql = "SELECT e.* FROM event e + WHERE e.tags::jsonb @> ?::jsonb + LIMIT 1"; + + $conn = $entityManager->getConnection(); + $result = $conn->executeQuery($sql, [ + json_encode([['d', $slug]]) + ]); + + $eventData = $result->fetchAssociative(); + + if ($eventData === false) { + throw new Exception('Category not found'); + } + + $tags = json_decode($eventData['tags'], true); $list = []; $coordinates = []; // Store full coordinates (kind:author:slug) $category = []; // Extract category metadata and article coordinates - foreach ($catIndex->getTags() as $tag) { + foreach ($tags as $tag) { if ($tag[0] === 'title') { $category['title'] = $tag[1]; } @@ -183,6 +197,10 @@ class DefaultController extends AbstractController } } + // Create a simple stdClass object with the tags for template compatibility + $catIndex = new \stdClass(); + $catIndex->tags = $tags; + return $this->render('pages/category.html.twig', [ 'mag' => $mag, 'magazine' => $magazine, diff --git a/src/Controller/MagazineWizardController.php b/src/Controller/MagazineWizardController.php index fe0bf24..35481d0 100644 --- a/src/Controller/MagazineWizardController.php +++ b/src/Controller/MagazineWizardController.php @@ -259,16 +259,25 @@ class MagazineWizardController extends AbstractController } #[Route('/magazine/wizard/edit/{slug}', name: 'mag_wizard_edit')] - public function editStart(string $slug, CacheInterface $redisCache, Request $request): Response + public function editStart(string $slug, EntityManagerInterface $entityManager, Request $request): Response { - // Load magazine event - $magEvent = $redisCache->get('magazine-' . $slug, function () { - return null; - }); - if (!$magEvent || !method_exists($magEvent, 'getTags')) { + // Load magazine event from database + $sql = "SELECT e.* FROM event e + WHERE e.tags::jsonb @> ?::jsonb + LIMIT 1"; + + $conn = $entityManager->getConnection(); + $result = $conn->executeQuery($sql, [ + json_encode([['d', $slug]]) + ]); + + $magEventData = $result->fetchAssociative(); + + if ($magEventData === false) { throw $this->createNotFoundException('Magazine not found'); } - $tags = (array)$magEvent->getTags(); + + $tags = json_decode($magEventData['tags'], true); $draft = new \App\Dto\MagazineDraft(); $draft->slug = $slug; @@ -285,9 +294,16 @@ class MagazineWizardController extends AbstractController $parts = explode(':', (string)$t[1], 3); if (count($parts) !== 3) { continue; } $catSlug = $parts[2]; - $catEvent = $redisCache->get('magazine-' . $catSlug, function () { return null; }); - if (!$catEvent || !method_exists($catEvent, 'getTags')) { continue; } - $ctags = (array)$catEvent->getTags(); + + // Query database for category event + $catResult = $conn->executeQuery($sql, [ + json_encode([['d', $catSlug]]) + ]); + $catEventData = $catResult->fetchAssociative(); + + if ($catEventData === false) { continue; } + + $ctags = json_decode($catEventData['tags'], true); $cat = new \App\Dto\CategoryDraft(); $cat->slug = $catSlug; $cat->title = $this->getTagValue($ctags, 'title') ?? ''; diff --git a/src/Controller/NzineController.php b/src/Controller/NzineController.php index 370b5dc..18dc3af 100644 --- a/src/Controller/NzineController.php +++ b/src/Controller/NzineController.php @@ -30,33 +30,101 @@ use Symfony\Component\String\Slugger\AsciiSlugger; class NzineController extends AbstractController { + /** + * List all NZines owned by the current user + */ + #[Route('/my-nzines', name: 'nzine_list')] + public function list(EntityManagerInterface $entityManager): Response + { + $user = $this->getUser(); + $nzines = []; + + if ($user) { + $userIdentifier = $user->getUserIdentifier(); + + // Find all nzines where the current user is the editor + $allNzines = $entityManager->getRepository(Nzine::class) + ->findBy(['editor' => $userIdentifier], ['id' => 'DESC']); + + foreach ($allNzines as $nzine) { + // Get the feed config for title and summary + $feedConfig = $nzine->getFeedConfig(); + $title = $feedConfig['title'] ?? 'Untitled NZine'; + $summary = $feedConfig['summary'] ?? null; + + // Count categories + $categoryCount = count($nzine->getMainCategories()); + + // Get main index to check publication status + $mainIndex = $entityManager->getRepository(EventEntity::class) + ->findOneBy([ + 'pubkey' => $nzine->getNpub(), + 'kind' => KindsEnum::PUBLICATION_INDEX->value, + // We'd need to filter by d-tag matching slug, but let's get first one for now + ]); + + $nzines[] = [ + 'id' => $nzine->getId(), + 'npub' => $nzine->getNpub(), + 'title' => $title, + 'summary' => $summary, + 'slug' => $nzine->getSlug(), + 'state' => $nzine->getState(), + 'categoryCount' => $categoryCount, + 'hasMainIndex' => $mainIndex !== null, + 'feedUrl' => $nzine->getFeedUrl(), + ]; + } + } + + return $this->render('nzine/list.html.twig', [ + 'nzines' => $nzines, + ]); + } + /** * @throws \JsonException */ #[Route('/nzine', name: 'nzine_index')] public function index(Request $request, NzineWorkflowService $nzineWorkflowService, EntityManagerInterface $entityManager): Response { - $form = $this->createForm(NzineBotType::class); - $form->handleRequest($request); $user = $this->getUser(); + $isAuthenticated = $user !== null; + + $form = $this->createForm(NzineBotType::class, null, [ + 'disabled' => !$isAuthenticated + ]); + $form->handleRequest($request); $nzine = $entityManager->getRepository(Nzine::class)->findAll(); - if ($form->isSubmitted() && $form->isValid()) { + if ($form->isSubmitted() && $form->isValid() && $isAuthenticated) { $data = $form->getData(); // init object $nzine = $nzineWorkflowService->init(); + + // Set RSS feed URL if provided + if (!empty($data['feedUrl'])) { + $nzine->setFeedUrl($data['feedUrl']); + } + + // Store title and summary for later use when creating main index + $nzine->setFeedConfig([ + 'title' => $data['name'], + 'summary' => $data['about'] + ]); + // create bot and nzine, save to persistence + // Note: We don't create the main index yet - that happens after categories are configured $nzine = $nzineWorkflowService->createProfile($nzine, $data['name'], $data['about'], $user); - // create main index - $nzineWorkflowService->createMainIndex($nzine, $data['name'], $data['about']); return $this->redirectToRoute('nzine_edit', ['npub' => $nzine->getNpub() ]); } return $this->render('pages/nzine-editor.html.twig', [ - 'form' => $form + 'form' => $form, + 'isAuthenticated' => $isAuthenticated ]); } @@ -84,9 +152,9 @@ class NzineController extends AbstractController }); $mainIndex = array_pop($mainIndexCandidates); - if (empty($mainIndex)) { - throw $this->createNotFoundException('Main index not found'); - } + + // If no main index exists yet, allow user to add categories but don't create indices yet + $canCreateIndices = !empty($mainIndex); $catForm = $this->createForm(NzineType::class, ['categories' => $nzine->getMainCategories()]); $catForm->handleRequest($request); @@ -95,6 +163,15 @@ class NzineController extends AbstractController // Process and normalize the 'tags' field $data = $catForm->get('categories')->getData(); + // Auto-generate slugs if not provided + $slugger = new AsciiSlugger(); + foreach ($data as &$cat) { + if (empty($cat['slug']) && !empty($cat['title'])) { + $cat['slug'] = $slugger->slug($cat['title'])->lower()->toString(); + } + } + unset($cat); // break reference + $nzine->setMainCategories($data); try { @@ -107,76 +184,105 @@ class NzineController extends AbstractController $managerRegistry->resetManager(); } - $catIndices = []; + // Only create category indices if main index exists + if ($canCreateIndices) { + $catIndices = []; + + $bot = $nzine->getNzineBot(); + $bot->setEncryptionService($encryptionService); + $private_key = $bot->getNsec(); // decrypted en route + + foreach ($data as $cat) { + // Validate category has required fields + if (!isset($cat['title']) || empty($cat['title'])) { + continue; // Skip invalid categories + } + + // check if such an index exists, only create new cats + $id = array_filter($indices, function ($k) use ($cat) { + return isset($cat['title']) && $cat['title'] === $k->getTitle(); + }); + if (!empty($id)) { continue; } + + // create new index + // currently not possible to edit existing, because there is no way to tell what has changed + // and which is the corresponding event + $title = $cat['title']; + $slug = isset($cat['slug']) && !empty($cat['slug']) + ? $cat['slug'] + : $slugger->slug($title)->lower()->toString(); + + // Use just the category slug for the d-tag so it can be found by the magazine frontend + // The main index will reference this via 'a' tags with full coordinates + $indexSlug = $slug; + + // create category index + $index = new Event(); + $index->setKind(KindsEnum::PUBLICATION_INDEX->value); + + $index->addTag(['d', $indexSlug]); + $index->addTag(['title', $title]); + $index->addTag(['auto-update', 'yes']); + $index->addTag(['type', 'magazine']); + + // Add tags for RSS matching + if (isset($cat['tags']) && is_array($cat['tags'])) { + foreach ($cat['tags'] as $tag) { + $index->addTag(['t', $tag]); + } + } + $index->setPublicKey($nzine->getNpub()); + + $signer = new Sign(); + $signer->signEvent($index, $private_key); + // save to persistence, first map to EventEntity + $serializer = new Serializer([new ObjectNormalizer()],[new JsonEncoder()]); + $i = $serializer->deserialize($index->toJson(), EventEntity::class, 'json'); + // don't save any more for now + $entityManager->persist($i); + $entityManager->flush(); + // TODO publish index to relays + + $catIndices[] = $index; + } - $bot = $nzine->getNzineBot(); - $bot->setEncryptionService($encryptionService); - $private_key = $bot->getNsec(); // decrypted en route - - foreach ($data as $cat) { - // check if such an index exists, only create new cats - $id = array_filter($indices, function ($k) use ($cat) { - return $cat['title'] === $k->getTitle(); - }); - if (!empty($id)) { continue; } - - // create new index - // currently not possible to edit existing, because there is no way to tell what has changed - // and which is the corresponding event - $slugger = new AsciiSlugger(); - $title = $cat['title']; - $slug = $mainIndex->getSlug().'-'.$slugger->slug($title)->lower(); - // create category index - $index = new Event(); - $index->setKind(KindsEnum::PUBLICATION_INDEX->value); - - $index->addTag(['d' => $slug]); - $index->addTag(['title' => $title]); - $index->addTag(['auto-update' => 'yes']); - $index->addTag(['type' => 'magazine']); - foreach ($cat['tags'] as $tag) { - $index->addTag(['t' => $tag]); + // add the new and updated indices to the main index + foreach ($catIndices as $idx) { + //remove e tags and add new + // $tags = array_splice($mainIndex->getTags(), -3); + // $mainIndex->setTags($tags); + // TODO add relay hints + $mainIndex->addTag(['a', KindsEnum::PUBLICATION_INDEX->value .':'. $idx->getPublicKey() .':'. $idx->getSlug()]); + // $mainIndex->addTag(['e' => $idx->getId()]); } - $index->setPublicKey($nzine->getNpub()); - $signer = new Sign(); - $signer->signEvent($index, $private_key); - // save to persistence, first map to EventEntity - $serializer = new Serializer([new ObjectNormalizer()],[new JsonEncoder()]); - $i = $serializer->deserialize($index->toJson(), EventEntity::class, 'json'); - // don't save any more for now - $entityManager->persist($i); + // re-sign main index and save to relays + // $signer = new Sign(); + // $signer->signEvent($mainIndex, $private_key); + // for now, just save new index $entityManager->flush(); - // TODO publish index to relays - - $catIndices[] = $index; + } else { + // Categories saved but no indices created yet + $this->addFlash('info', 'Categories saved. Indices will be created once the main index is published.'); } - // add the new and updated indices to the main index - foreach ($catIndices as $idx) { - //remove e tags and add new - // $tags = array_splice($mainIndex->getTags(), -3); - // $mainIndex->setTags($tags); - // TODO add relay hints - $mainIndex->addTag(['a' => KindsEnum::PUBLICATION_INDEX->value .':'. $idx->getPublicKey() .':'. $idx->getSlug()]); - // $mainIndex->addTag(['e' => $idx->getId() ]); + // redirect to route nzine_view if main index exists, otherwise stay on edit page + if ($canCreateIndices) { + return $this->redirectToRoute('nzine_view', [ + 'npub' => $nzine->getNpub(), + ]); + } else { + return $this->redirectToRoute('nzine_edit', [ + 'npub' => $nzine->getNpub(), + ]); } - - // re-sign main index and save to relays - // $signer = new Sign(); - // $signer->signEvent($mainIndex, $private_key); - // for now, just save new index - $entityManager->flush(); - - // redirect to route nzine_view - return $this->redirectToRoute('nzine_view', [ - 'npub' => $nzine->getNpub(), - ]); } return $this->render('pages/nzine-editor.html.twig', [ 'nzine' => $nzine, 'indices' => $indices, + 'mainIndex' => $mainIndex, + 'canCreateIndices' => $canCreateIndices, 'bot' => $bot ?? null, // if null, the profile for the bot doesn't exist yet 'catForm' => $catForm ]); @@ -256,4 +362,149 @@ class NzineController extends AbstractController ]); } + #[Route('/nzine/{npub}/publish', name: 'nzine_publish', methods: ['POST'])] + public function publish($npub, EntityManagerInterface $entityManager, + EncryptionService $encryptionService, + ManagerRegistry $managerRegistry): Response + { + $nzine = $entityManager->getRepository(Nzine::class)->findOneBy(['npub' => $npub]); + if (!$nzine) { + throw $this->createNotFoundException('N-Zine not found'); + } + + // Check if categories are configured + if (empty($nzine->getMainCategories())) { + $this->addFlash('error', 'Please add at least one category before publishing.'); + return $this->redirectToRoute('nzine_edit', ['npub' => $npub]); + } + + // Check if main index already exists + $indices = $entityManager->getRepository(EventEntity::class)->findBy(['pubkey' => $npub, 'kind' => KindsEnum::PUBLICATION_INDEX]); + $mainIndexCandidates = array_filter($indices, function ($index) use ($nzine) { + return $index->getSlug() == $nzine->getSlug(); + }); + + if (!empty($mainIndexCandidates)) { + $this->addFlash('warning', 'Main index already exists.'); + return $this->redirectToRoute('nzine_edit', ['npub' => $npub]); + } + + try { + // Start transaction + $entityManager->beginTransaction(); + + $bot = $nzine->getNzineBot(); + if (!$bot) { + throw new \RuntimeException('Nzine bot not found'); + } + + $bot->setEncryptionService($encryptionService); + $private_key = $bot->getNsec(); + + if (!$private_key) { + throw new \RuntimeException('Failed to decrypt bot private key'); + } + + // Get title and summary from feedConfig + $config = $nzine->getFeedConfig(); + $title = $config['title'] ?? 'Untitled'; + $summary = $config['summary'] ?? ''; + + // Generate slug for main index + $slugger = new AsciiSlugger(); + $slug = 'nzine-'.$slugger->slug($title)->lower().'-'.rand(10000,99999); + $nzine->setSlug($slug); + + // Create main index + $mainIndex = new Event(); + $mainIndex->setKind(KindsEnum::PUBLICATION_INDEX->value); + $mainIndex->addTag(['d', $slug]); + $mainIndex->addTag(['title', $title]); + $mainIndex->addTag(['summary', $summary]); + $mainIndex->addTag(['auto-update', 'yes']); + $mainIndex->addTag(['type', 'magazine']); + $mainIndex->setPublicKey($nzine->getNpub()); + + // Create category indices + $catIndices = []; + foreach ($nzine->getMainCategories() as $cat) { + if (!isset($cat['title'])) { + continue; // Skip categories without titles + } + + $catTitle = $cat['title']; + $catSlug = $cat['slug'] ?? $slugger->slug($catTitle)->lower()->toString(); + // Use just the category slug for the d-tag so it can be found by the magazine frontend + // The main index will reference this via 'a' tags with full coordinates + $indexSlug = $catSlug; + + $catIndex = new Event(); + $catIndex->setKind(KindsEnum::PUBLICATION_INDEX->value); + $catIndex->addTag(['d', $indexSlug]); + $catIndex->addTag(['title', $catTitle]); + $catIndex->addTag(['auto-update', 'yes']); + $catIndex->addTag(['type', 'magazine']); + + // Add tags for RSS matching + if (isset($cat['tags']) && is_array($cat['tags'])) { + foreach ($cat['tags'] as $tag) { + $catIndex->addTag(['t', $tag]); + } + } + $catIndex->setPublicKey($nzine->getNpub()); + + // Sign category index + $signer = new Sign(); + $signer->signEvent($catIndex, $private_key); + + // Save category index + $serializer = new Serializer([new ObjectNormalizer()],[new JsonEncoder()]); + $i = $serializer->deserialize($catIndex->toJson(), EventEntity::class, 'json'); + $entityManager->persist($i); + + // Add reference to main index + $mainIndex->addTag(['a', KindsEnum::PUBLICATION_INDEX->value .':'. $catIndex->getPublicKey() .':'. $indexSlug]); + + $catIndices[] = $catIndex; + } + + // Sign main index (after adding all category references) + $signer = new Sign(); + $signer->signEvent($mainIndex, $private_key); + + // Save main index + $serializer = new Serializer([new ObjectNormalizer()],[new JsonEncoder()]); + $mainIndexEntity = $serializer->deserialize($mainIndex->toJson(), EventEntity::class, 'json'); + $entityManager->persist($mainIndexEntity); + + // Update nzine state + $nzine->setState('published'); + $entityManager->persist($nzine); + + // Commit transaction + $entityManager->flush(); + $entityManager->commit(); + + $this->addFlash('success', sprintf( + 'N-Zine published successfully! Created main index and %d category indices.', + count($catIndices) + )); + + return $this->redirectToRoute('nzine_edit', ['npub' => $npub]); + + } catch (Exception $e) { + if ($entityManager->getConnection()->isTransactionActive()) { + $entityManager->rollback(); + } + $managerRegistry->resetManager(); + + $this->addFlash('error', 'Failed to publish N-Zine: ' . $e->getMessage()); + + // Log the full error for debugging + error_log('N-Zine publish error: ' . $e->getMessage() . "\n" . $e->getTraceAsString()); + + return $this->redirectToRoute('nzine_edit', ['npub' => $npub]); + } + } + } diff --git a/src/Entity/Article.php b/src/Entity/Article.php index bb86536..8a6889f 100644 --- a/src/Entity/Article.php +++ b/src/Entity/Article.php @@ -257,6 +257,12 @@ class Article return $this; } + public function clearTopics(): static + { + $this->topics = []; + return $this; + } + public function getEventStatus(): ?EventStatusEnum { return $this->eventStatus; diff --git a/src/Entity/Event.php b/src/Entity/Event.php index f71568c..6e4918c 100644 --- a/src/Entity/Event.php +++ b/src/Entity/Event.php @@ -135,7 +135,7 @@ class Event public function getSlug(): ?string { foreach ($this->getTags() as $tag) { - if ($tag[0] === 'd') { + if (key_exists(0, $tag) && $tag[0] === 'd') { return $tag[1]; } } diff --git a/src/Entity/Nzine.php b/src/Entity/Nzine.php index d623359..9d6093c 100644 --- a/src/Entity/Nzine.php +++ b/src/Entity/Nzine.php @@ -41,6 +41,15 @@ class Nzine #[ORM\Column(type: 'string')] private string $state = 'draft'; + #[ORM\Column(type: Types::TEXT, nullable: true)] + private ?string $feedUrl = null; + + #[ORM\Column(type: Types::DATETIME_IMMUTABLE, nullable: true)] + private ?\DateTimeImmutable $lastFetchedAt = null; + + #[ORM\Column(type: Types::JSON, nullable: true)] + private ?array $feedConfig = null; + public function __construct() { $this->mainCategories = new ArrayCollection(); @@ -138,4 +147,34 @@ class Nzine { $this->state = $state; } + + public function getFeedUrl(): ?string + { + return $this->feedUrl; + } + + public function setFeedUrl(?string $feedUrl): void + { + $this->feedUrl = $feedUrl; + } + + public function getLastFetchedAt(): ?\DateTimeImmutable + { + return $this->lastFetchedAt; + } + + public function setLastFetchedAt(?\DateTimeImmutable $lastFetchedAt): void + { + $this->lastFetchedAt = $lastFetchedAt; + } + + public function getFeedConfig(): ?array + { + return $this->feedConfig; + } + + public function setFeedConfig(?array $feedConfig): void + { + $this->feedConfig = $feedConfig; + } } diff --git a/src/Examples/RssNzineSetupExample.php b/src/Examples/RssNzineSetupExample.php new file mode 100644 index 0000000..7373d94 --- /dev/null +++ b/src/Examples/RssNzineSetupExample.php @@ -0,0 +1,159 @@ +nzineRepository->find($nzineId); + + if (!$nzine) { + throw new \RuntimeException("Nzine not found: $nzineId"); + } + + // Set the RSS feed URL + $nzine->setFeedUrl('https://example.com/feed.rss'); + + // Configure categories with tags for RSS item matching + $categories = [ + [ + 'name' => 'Artificial Intelligence', + 'slug' => 'ai', + 'tags' => ['artificial-intelligence', 'machine-learning', 'AI', 'ML', 'deep-learning', 'neural-networks'] + ], + [ + 'name' => 'Blockchain & Crypto', + 'slug' => 'blockchain', + 'tags' => ['crypto', 'cryptocurrency', 'blockchain', 'bitcoin', 'ethereum', 'web3', 'defi', 'nft'] + ], + [ + 'name' => 'Programming', + 'slug' => 'programming', + 'tags' => ['programming', 'coding', 'development', 'software', 'javascript', 'python', 'rust', 'go'] + ], + [ + 'name' => 'Nostr Protocol', + 'slug' => 'nostr', + 'tags' => ['nostr', 'decentralized', 'social-media', 'protocol'] + ] + ]; + + $nzine->setMainCategories($categories); + + // Optional: Set custom feed configuration + $nzine->setFeedConfig([ + 'enabled' => true, + 'description' => 'Tech news aggregator', + // Future options: + // 'max_age_days' => 7, + // 'fetch_full_content' => true, + ]); + + $this->entityManager->flush(); + + echo "RSS feed configured for nzine #{$nzineId}\n"; + echo "Feed URL: " . $nzine->getFeedUrl() . "\n"; + echo "Categories: " . count($categories) . "\n"; + } + + /** + * Example: Create a new RSS-enabled nzine from scratch + */ + public function createRssEnabledNzine( + string $title, + string $summary, + string $feedUrl, + array $categories + ): Nzine { + // Note: This is a simplified example. In practice, you should: + // 1. Use NzineWorkflowService to create the bot and profile + // 2. Create the main index + // 3. Create nested indices + // 4. Transition through the workflow states + + $nzine = new Nzine(); + $nzine->setFeedUrl($feedUrl); + $nzine->setMainCategories($categories); + + // You would normally use the workflow service here: + // $this->nzineWorkflowService->init($nzine); + // $this->nzineWorkflowService->createProfile(...); + // etc. + + $this->entityManager->persist($nzine); + $this->entityManager->flush(); + + return $nzine; + } + + /** + * Example: List all RSS-enabled nzines + */ + public function listRssNzines(): void + { + $nzines = $this->nzineRepository->findActiveRssNzines(); + + echo "RSS-enabled Nzines:\n"; + echo str_repeat("=", 80) . "\n"; + + foreach ($nzines as $nzine) { + echo sprintf( + "ID: %d | Slug: %s | Feed: %s\n", + $nzine->getId(), + $nzine->getSlug() ?? 'N/A', + $nzine->getFeedUrl() + ); + + $lastFetched = $nzine->getLastFetchedAt(); + if ($lastFetched) { + echo " Last fetched: " . $lastFetched->format('Y-m-d H:i:s') . "\n"; + } + + echo " Categories: " . count($nzine->getMainCategories()) . "\n"; + echo "\n"; + } + } + + /** + * Example RSS feed URLs for testing + */ + public static function getExampleFeeds(): array + { + return [ + 'tech' => [ + 'TechCrunch' => 'https://techcrunch.com/feed/', + 'Hacker News' => 'https://hnrss.org/newest', + 'Ars Technica' => 'https://feeds.arstechnica.com/arstechnica/index', + ], + 'crypto' => [ + 'CoinDesk' => 'https://www.coindesk.com/arc/outboundfeeds/rss/', + 'Bitcoin Magazine' => 'https://bitcoinmagazine.com/.rss/full/', + ], + 'programming' => [ + 'Dev.to' => 'https://dev.to/feed', + 'GitHub Blog' => 'https://github.blog/feed/', + ] + ]; + } +} + diff --git a/src/Form/MainCategoryType.php b/src/Form/MainCategoryType.php index ab8b461..1763bda 100644 --- a/src/Form/MainCategoryType.php +++ b/src/Form/MainCategoryType.php @@ -21,9 +21,19 @@ class MainCategoryType extends AbstractType $builder ->add('title', TextType::class, [ 'label' => 'Title', + 'help' => 'Category display name' + ]) + ->add('slug', TextType::class, [ + 'label' => 'Slug', + 'help' => 'URL-friendly identifier (e.g., ai-ml, blockchain)', + 'required' => false, + 'attr' => [ + 'placeholder' => 'auto-generated from title if empty' + ] ]) ->add('tags', TextType::class, [ 'label' => 'Tags', + 'help' => 'Comma-separated tags for matching RSS items (e.g., artificial-intelligence, AI, machine-learning)' ]); $builder->get('tags')->addModelTransformer($this->transformer); diff --git a/src/Form/NzineBotType.php b/src/Form/NzineBotType.php index f9dc068..b69619e 100644 --- a/src/Form/NzineBotType.php +++ b/src/Form/NzineBotType.php @@ -15,14 +15,38 @@ class NzineBotType extends AbstractType { public function buildForm(FormBuilderInterface $builder, array $options): void { + $isDisabled = $options['disabled'] ?? false; + $builder ->add('name', TextType::class, [ - 'required' => true + 'required' => true, + 'label' => 'N-Zine Name', + 'help' => 'The name of your N-Zine publication', + 'disabled' => $isDisabled ]) ->add('about', TextareaType::class, [ - 'required' => false + 'required' => false, + 'label' => 'Description', + 'help' => 'Describe what this N-Zine is about', + 'disabled' => $isDisabled + ]) + ->add('feedUrl', TextType::class, [ + 'required' => false, + 'label' => 'RSS Feed URL', + 'help' => 'Optional: Add an RSS/Atom feed URL to automatically fetch and publish articles', + 'attr' => [ + 'placeholder' => 'https://example.com/feed.rss' + ], + 'disabled' => $isDisabled + ]) + ->add('submit', SubmitType::class, [ + 'label' => 'Create N-Zine', + 'disabled' => $isDisabled, + 'attr' => [ + 'class' => 'btn btn-primary', + 'title' => $isDisabled ? 'Please login to create an N-Zine' : '' + ] ]) - ->add('submit', SubmitType::class) ; } diff --git a/src/Repository/NzineRepository.php b/src/Repository/NzineRepository.php index 30c6988..ce22361 100644 --- a/src/Repository/NzineRepository.php +++ b/src/Repository/NzineRepository.php @@ -16,6 +16,35 @@ class NzineRepository extends ServiceEntityRepository parent::__construct($registry, Nzine::class); } + /** + * Find all nzines with RSS feeds configured and in published state + * + * @return Nzine[] + */ + public function findActiveRssNzines(): array + { + return $this->createQueryBuilder('n') + ->where('n.feedUrl IS NOT NULL') + ->andWhere('n.state = :state') + ->setParameter('state', 'published') + ->orderBy('n.id', 'ASC') + ->getQuery() + ->getResult(); + } + + /** + * Find a specific nzine by ID with RSS feed configured + */ + public function findRssNzineById(int $id): ?Nzine + { + return $this->createQueryBuilder('n') + ->where('n.id = :id') + ->andWhere('n.feedUrl IS NOT NULL') + ->setParameter('id', $id) + ->getQuery() + ->getOneOrNullResult(); + } + // /** // * @return Nzine[] Returns an array of Nzine objects // */ diff --git a/src/Service/RssFeedService.php b/src/Service/RssFeedService.php new file mode 100644 index 0000000..a0bd8fe --- /dev/null +++ b/src/Service/RssFeedService.php @@ -0,0 +1,189 @@ +logger->info('Fetching RSS feed', ['url' => $feedUrl]); + + $response = $this->httpClient->request('GET', $feedUrl, [ + 'timeout' => 30, + 'headers' => [ + 'User-Agent' => 'Newsroom RSS Aggregator/1.0', + ], + ]); + + if ($response->getStatusCode() !== 200) { + throw new \Exception(sprintf('HTTP error %d when fetching feed', $response->getStatusCode())); + } + + $xmlContent = $response->getContent(); + $items = $this->parseRssFeed($xmlContent); + + $this->logger->info('RSS feed fetched successfully', [ + 'url' => $feedUrl, + 'items' => count($items), + ]); + + return $items; + } catch (\Exception $e) { + $this->logger->error('Failed to fetch RSS feed', [ + 'url' => $feedUrl, + 'error' => $e->getMessage(), + ]); + throw $e; + } + } + + /** + * Parse RSS XML content into structured array + * + * @param string $xmlContent Raw XML content + * @return array Array of parsed feed items + * @throws \Exception if XML parsing fails + */ + private function parseRssFeed(string $xmlContent): array + { + libxml_use_internal_errors(true); + $xml = simplexml_load_string($xmlContent); + + if ($xml === false) { + $errors = libxml_get_errors(); + libxml_clear_errors(); + throw new \Exception('Failed to parse RSS XML: ' . json_encode($errors)); + } + + $items = []; + + // Handle both RSS 2.0 and Atom feeds + if (isset($xml->channel->item)) { + // RSS 2.0 + foreach ($xml->channel->item as $item) { + $items[] = $this->parseRssItem($item); + } + } elseif (isset($xml->entry)) { + // Atom feed + foreach ($xml->entry as $entry) { + $items[] = $this->parseAtomEntry($entry); + } + } + + return $items; + } + + /** + * Parse a single RSS 2.0 item + */ + private function parseRssItem(\SimpleXMLElement $item): array + { + $namespaces = $item->getNamespaces(true); + $content = ''; + + // Try to get full content from content:encoded or description + if (isset($namespaces['content'])) { + $contentChildren = $item->children($namespaces['content']); + if (isset($contentChildren->encoded)) { + $content = (string) $contentChildren->encoded; + } + } + + if (empty($content)) { + $content = (string) ($item->description ?? ''); + } + + // Extract categories + $categories = []; + if (isset($item->category)) { + foreach ($item->category as $category) { + $categories[] = (string) $category; + } + } + + // Parse publication date + $pubDate = null; + if (isset($item->pubDate)) { + $pubDate = new \DateTimeImmutable((string) $item->pubDate); + } + + return [ + 'title' => (string) ($item->title ?? ''), + 'link' => (string) ($item->link ?? ''), + 'pubDate' => $pubDate, + 'description' => (string) ($item->description ?? ''), + 'content' => $content, + 'categories' => $categories, + 'guid' => (string) ($item->guid ?? ''), + ]; + } + + /** + * Parse a single Atom entry + */ + private function parseAtomEntry(\SimpleXMLElement $entry): array + { + $namespaces = $entry->getNamespaces(true); + + // Get link + $link = ''; + if (isset($entry->link)) { + foreach ($entry->link as $l) { + if ((string) $l['rel'] === 'alternate' || !isset($l['rel'])) { + $link = (string) $l['href']; + break; + } + } + } + + // Get content + $content = (string) ($entry->content ?? $entry->summary ?? ''); + + // Get categories/tags + $categories = []; + if (isset($entry->category)) { + foreach ($entry->category as $category) { + $categories[] = (string) $category['term']; + } + } + + // Parse publication date + $pubDate = null; + if (isset($entry->published)) { + $pubDate = new \DateTimeImmutable((string) $entry->published); + } elseif (isset($entry->updated)) { + $pubDate = new \DateTimeImmutable((string) $entry->updated); + } + + return [ + 'title' => (string) ($entry->title ?? ''), + 'link' => $link, + 'pubDate' => $pubDate, + 'description' => (string) ($entry->summary ?? ''), + 'content' => $content, + 'categories' => $categories, + 'guid' => (string) ($entry->id ?? ''), + ]; + } +} + diff --git a/src/Service/RssToNostrConverter.php b/src/Service/RssToNostrConverter.php new file mode 100644 index 0000000..c08250d --- /dev/null +++ b/src/Service/RssToNostrConverter.php @@ -0,0 +1,166 @@ +getNzineBot(); + if (!$bot) { + throw new \RuntimeException('Nzine bot not found'); + } + + $bot->setEncryptionService($this->encryptionService); + $privateKey = $bot->getNsec(); + + if (!$privateKey) { + throw new \RuntimeException('Bot private key not found'); + } + + // Create the event + $event = new Event(); + $event->setKind(KindsEnum::LONGFORM->value); + + // Set content + $content = $rssItem['content'] ?? $rssItem['description'] ?? ''; + + // If we have both content and link, append a reference + if (!empty($rssItem['link'])) { + $content .= "\n\n---\n\nOriginal article: " . $rssItem['link']; + } + + $event->setContent($content); + + // Generate unique slug from title and timestamp + $slug = $this->generateSlug($rssItem['title'], $rssItem['pubDate']); + $event->addTag(['d', $slug]); + + // Add title tag + if (!empty($rssItem['title'])) { + $event->addTag(['title', $rssItem['title']]); + } + + // Add summary tag + if (!empty($rssItem['description'])) { + $summary = $this->htmlToPlainText($rssItem['description']); + $event->addTag(['summary', $summary]); + } + + // Add published_at tag + if ($rssItem['pubDate'] instanceof \DateTimeImmutable) { + $event->addTag(['published_at', (string) $rssItem['pubDate']->getTimestamp()]); + } + + // Add category tag (t tag) - only if category matched + if ($matchedCategory && isset($matchedCategory['slug'])) { + $event->addTag(['t', $matchedCategory['slug']]); + } + + // Add reference to original URL + if (!empty($rssItem['link'])) { + $event->addTag(['r', $rssItem['link']]); + } + + // Add reference to category index if provided and category matched + if ($categoryIndexEventId && $matchedCategory && isset($matchedCategory['slug'])) { + $npub = $nzine->getNpub(); + $event->addTag(['a', KindsEnum::PUBLICATION_INDEX->value . ':' . $npub . ':' . $matchedCategory['slug']]); + } + + // Add client tag to indicate source + $event->addTag(['client', 'newsroom-rss-aggregator']); + + // Sign the event + $signer = new Sign(); + $signer->signEvent($event, $privateKey); + + $this->logger->info('Created Nostr event from RSS item', [ + 'title' => $rssItem['title'], + 'slug' => $slug, + 'category' => $matchedCategory['name'] ?? null, + ]); + + return $event; + } + + /** + * Generate a unique slug from title and timestamp + */ + private function generateSlug(string $title, ?\DateTimeImmutable $pubDate): string + { + $slugger = new AsciiSlugger(); + $baseSlug = $slugger->slug($title)->lower()->toString(); + + // Limit base slug length + if (strlen($baseSlug) > 50) { + $baseSlug = substr($baseSlug, 0, 50); + } + + // Add timestamp for uniqueness + $timestamp = $pubDate ? $pubDate->format('Y-m-d-His') : date('Y-m-d-His'); + + return $baseSlug . '-' . $timestamp; + } + + /** + * Convert HTML content to plain text + * Strips HTML tags and decodes HTML entities + */ + private function htmlToPlainText(?string $html): string + { + if (empty($html)) { + return ''; + } + + // Strip HTML tags + $text = strip_tags($html); + + // Decode HTML entities + $text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'); + + // Normalize whitespace + $text = preg_replace('/\s+/', ' ', $text); + + // Trim + return trim($text); + } + + /** + * Check if a slug already exists in the database + * This is used by the command to detect duplicates + */ + public function generateSlugForItem(array $rssItem): string + { + return $this->generateSlug($rssItem['title'], $rssItem['pubDate']); + } +} diff --git a/src/Service/TagMatchingService.php b/src/Service/TagMatchingService.php new file mode 100644 index 0000000..bab805c --- /dev/null +++ b/src/Service/TagMatchingService.php @@ -0,0 +1,84 @@ +logger->debug('Category match found', [ + 'rss_category' => $rssCategory, + 'nzine_category' => $nzineCategory['name'] ?? $nzineCategory['title'] ?? $nzineCategory['slug'] ?? 'unknown', + ]); + return $nzineCategory; + } + } + } + + $this->logger->debug('No category match found', [ + 'rss_categories' => $rssItemCategories, + ]); + + return null; + } + + /** + * Extract all unique tags from nzine categories + * + * @param array $nzineCategories Array of nzine categories + * @return array Array of all unique tags + */ + public function extractAllTags(array $nzineCategories): array + { + $allTags = []; + + foreach ($nzineCategories as $category) { + if (!isset($category['tags']) || empty($category['tags'])) { + continue; + } + + $tags = is_array($category['tags']) + ? $category['tags'] + : explode(',', $category['tags']); + + $allTags = array_merge($allTags, array_map('trim', $tags)); + } + + return array_unique($allTags); + } +} diff --git a/src/Twig/Components/Header.php b/src/Twig/Components/Header.php index 934be0d..857134b 100644 --- a/src/Twig/Components/Header.php +++ b/src/Twig/Components/Header.php @@ -11,21 +11,4 @@ use Symfony\UX\TwigComponent\Attribute\AsTwigComponent; #[AsTwigComponent] class Header { - public array $cats; - - /** - * @throws InvalidArgumentException - */ - public function __construct(private readonly CacheInterface $redisCache) - { - $mag = $this->redisCache->get('magazine-newsroom-magazine-by-newsroom', function (){ - return null; - }); - - $tags = $mag->getTags(); - - $this->cats = array_filter($tags, function($tag) { - return ($tag[0] === 'a'); - }); - } } diff --git a/src/Twig/Components/Molecules/CategoryLink.php b/src/Twig/Components/Molecules/CategoryLink.php index 7d4ecf4..087ac98 100644 --- a/src/Twig/Components/Molecules/CategoryLink.php +++ b/src/Twig/Components/Molecules/CategoryLink.php @@ -2,7 +2,8 @@ namespace App\Twig\Components\Molecules; -use Symfony\Contracts\Cache\CacheInterface; +use App\Entity\Event; +use Doctrine\ORM\EntityManagerInterface; use Symfony\UX\TwigComponent\Attribute\AsTwigComponent; #[AsTwigComponent] @@ -12,7 +13,7 @@ final class CategoryLink public string $slug; public ?string $mag = null; // magazine slug passed from parent (optional) - public function __construct(private CacheInterface $redisCache) + public function __construct(private EntityManagerInterface $entityManager) { } @@ -22,16 +23,25 @@ final class CategoryLink $parts = explode(':', $coordinate[1], 3); // Expect format kind:pubkey:slug $this->slug = $parts[2] ?? ''; - $cat = $this->redisCache->get('magazine-' . $this->slug, function (){ - return null; - }); - if ($cat === null) { + // Query the database for the category event by slug using native SQL + $sql = "SELECT e.* FROM event e + WHERE e.tags::jsonb @> ?::jsonb + LIMIT 1"; + + $conn = $this->entityManager->getConnection(); + $result = $conn->executeQuery($sql, [ + json_encode([['d', $this->slug]]) + ]); + + $eventData = $result->fetchAssociative(); + + if ($eventData === false) { $this->title = $this->slug ?: 'Category'; return; } - $tags = $cat->getTags(); + $tags = json_decode($eventData['tags'], true); $title = array_filter($tags, function($tag) { return ($tag[0] === 'title'); diff --git a/src/Twig/Components/Organisms/FeaturedList.php b/src/Twig/Components/Organisms/FeaturedList.php index a130d50..85702df 100644 --- a/src/Twig/Components/Organisms/FeaturedList.php +++ b/src/Twig/Components/Organisms/FeaturedList.php @@ -2,12 +2,10 @@ namespace App\Twig\Components\Organisms; +use Doctrine\ORM\EntityManagerInterface; use Elastica\Query; use Elastica\Query\Terms; use FOS\ElasticaBundle\Finder\FinderInterface; -use Psr\Cache\InvalidArgumentException; -use swentel\nostr\Event\Event; -use Symfony\Contracts\Cache\CacheInterface; use Symfony\UX\TwigComponent\Attribute\AsTwigComponent; #[AsTwigComponent] @@ -20,25 +18,39 @@ final class FeaturedList public array $list = []; public function __construct( - private readonly CacheInterface $redisCache, + private readonly EntityManagerInterface $entityManager, private readonly FinderInterface $finder) { } /** - * @throws InvalidArgumentException * @throws \Exception */ public function mount($category): void { $parts = explode(':', $category[1]); - /** @var Event $catIndex */ - $catIndex = $this->redisCache->get('magazine-' . $parts[2], function (){ - throw new \Exception('Not found'); - }); + $categorySlug = $parts[2] ?? ''; + + // Query the database for the category event by slug using native SQL + $sql = "SELECT e.* FROM event e + WHERE e.tags::jsonb @> ?::jsonb + LIMIT 1"; + + $conn = $this->entityManager->getConnection(); + $result = $conn->executeQuery($sql, [ + json_encode([['d', $categorySlug]]) + ]); + + $eventData = $result->fetchAssociative(); + + if ($eventData === false) { + return; + } + + $tags = json_decode($eventData['tags'], true); $slugs = []; - foreach ($catIndex->getTags() as $tag) { + foreach ($tags as $tag) { if ($tag[0] === 'title') { $this->title = $tag[1]; } diff --git a/templates/admin/magazines.html.twig b/templates/admin/magazines.html.twig index 6ed6328..dcd6eeb 100644 --- a/templates/admin/magazines.html.twig +++ b/templates/admin/magazines.html.twig @@ -1,7 +1,9 @@ {% extends 'layout.html.twig' %} {% block body %} -

Magazines

+
+

Magazines

+
{% if magazines is empty %}

No magazines found.

diff --git a/templates/admin/magazines_orphaned.html.twig b/templates/admin/magazines_orphaned.html.twig new file mode 100644 index 0000000..f5f1d6c --- /dev/null +++ b/templates/admin/magazines_orphaned.html.twig @@ -0,0 +1,64 @@ +{% extends 'layout.html.twig' %} + +{% block body %} + +

N-Zines

+

Clean up broken or incomplete N-Zine data from the database.

+ +
+ +

Malformed N-Zines

+

These have Nzine entities but missing or broken indices:

+ + {% if malformed is empty %} +
✓ No malformed nzines found!
+ {% else %} +
+ Found {{ malformed|length }} malformed nzine(s). These should be deleted or fixed. +
+ + + + + + + + + + + + + + {% for item in malformed %} + + + + + + {% endfor %} + +
NPubSlugStateCategoriesIndicesActions
{{ item.npub|slice(0, 16) }}... + {% if item.slug %} + {{ item.slug }} + {% else %} + NULL + {% endif %} + +
+ +
+
+ {% endif %} + +
+ +
+ Tips: + +
+{% endblock %} + diff --git a/templates/components/Organisms/FeaturedList.html.twig b/templates/components/Organisms/FeaturedList.html.twig index 030117b..e616add 100644 --- a/templates/components/Organisms/FeaturedList.html.twig +++ b/templates/components/Organisms/FeaturedList.html.twig @@ -1,6 +1,6 @@ -
+
{% if list %} -