Rewire cache and sync

docker/cron/crontab

@@ -1,3 +1,19 @@
-0 */6 * * * /index_articles.sh >> /var/log/cron.log 2>&1
+# Article Processing Cron Jobs
-2 */2 * * * /media_discovery.sh >> /var/log/cron.log 2>&1
+# ========================================
-0 */2 * * * /article_discovery.sh >> /var/log/cron.log 2>&1
+
+# Post-process articles (QA, indexing, mark as indexed)
+# Runs every 5 minutes to process articles ingested by hydration worker
+*/5 * * * * /var/www/html/docker/cron/post_process_articles.sh >> /var/log/cron-post-process.log 2>&1
+
+# Backfill historical articles
+# Runs once daily at 2 AM for historical data
+0 2 * * * /var/www/html/docker/cron/index_articles.sh >> /var/log/cron-backfill.log 2>&1
+
+# Cache latest articles for Redis views
+# Runs every 15 minutes to keep cache fresh
+*/15 * * * * php /var/www/html/bin/console app:cache_latest_articles >> /var/log/cron-cache-articles.log 2>&1
+
+# Cache latest highlights for Redis views
+# Runs every 30 minutes
+*/30 * * * * php /var/www/html/bin/console app:cache-latest-highlights >> /var/log/cron-cache-highlights.log 2>&1
+

docker/cron/index_articles.sh

@@ -2,8 +2,21 @@
 set -e
 export PATH="/usr/local/bin:/usr/bin:/bin"
 
-# Run Symfony commands sequentially
+# ========================================
+# BACKFILL ONLY - FOR HISTORICAL ARTICLES
+# ========================================
+# This cron ONLY fetches historical articles for backfill purposes.
+# Run once daily or as needed for historical data.
+#
+# NEW: Post-processing (QA, indexing) now runs via separate cron
+# See: docker/cron/post_process_articles.sh (runs every few minutes)
+# ========================================
+
+echo "$(date '+%Y-%m-%d %H:%M:%S') - Starting article backfill..."
+
+# Backfill: Fetch articles from last week (only needed for historical data)
 php /var/www/html/bin/console articles:get -- '-1 week' 'now'
-php /var/www/html/bin/console articles:qa
+
-php /var/www/html/bin/console articles:index
+echo "$(date '+%Y-%m-%d %H:%M:%S') - Article backfill completed"
-php /var/www/html/bin/console articles:indexed
+
+# Note: QA and indexing are handled by post_process_articles.sh (separate cron)

docker/cron/post_process_articles.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+export PATH="/usr/local/bin:/usr/bin:/bin"
+
+# ========================================
+# ARTICLE POST-PROCESSING CRON
+# ========================================
+# Runs every few minutes to process articles that need QA and indexing
+# Replaces the old articles:get + qa + index + indexed sequence
+# Now only runs post-processing since hydration worker handles ingestion
+# ========================================
+
+echo "$(date '+%Y-%m-%d %H:%M:%S') - Starting article post-processing..."
+
+# Run post-processing commands (QA, index, mark as indexed)
+php /var/www/html/bin/console articles:post-process
+
+echo "$(date '+%Y-%m-%d %H:%M:%S') - Article post-processing completed"
+

src/Command/ArticlePostProcessCommand.php

@@ -0,0 +1,135 @@
+<?php
+
+namespace App\Command;
+
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\Process\Process;
+
+#[AsCommand(
+    name: 'articles:post-process',
+    description: 'Run post-processing commands (QA, index, indexed) on articles'
+)]
+class ArticlePostProcessCommand extends Command
+{
+    protected function configure(): void
+    {
+        $this
+            ->addOption(
+                'skip-qa',
+                null,
+                InputOption::VALUE_NONE,
+                'Skip the QA step'
+            )
+            ->addOption(
+                'skip-index',
+                null,
+                InputOption::VALUE_NONE,
+                'Skip the ElasticSearch indexing step'
+            )
+            ->addOption(
+                'skip-indexed',
+                null,
+                InputOption::VALUE_NONE,
+                'Skip marking articles as indexed'
+            );
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $io = new SymfonyStyle($input, $output);
+
+        $io->title('Article Post-Processing');
+        $io->text('Running QA and indexing commands sequentially...');
+        $io->newLine();
+
+        $skipQa = $input->getOption('skip-qa');
+        $skipIndex = $input->getOption('skip-index');
+        $skipIndexed = $input->getOption('skip-indexed');
+
+        $commands = [];
+        if (!$skipQa) {
+            $commands[] = [
+                'name' => 'articles:qa',
+                'description' => 'Quality Assurance',
+            ];
+        }
+        if (!$skipIndex) {
+            $commands[] = [
+                'name' => 'articles:index',
+                'description' => 'ElasticSearch Indexing',
+            ];
+        }
+        if (!$skipIndexed) {
+            $commands[] = [
+                'name' => 'articles:indexed',
+                'description' => 'Mark as Indexed',
+            ];
+        }
+
+        if (empty($commands)) {
+            $io->warning('All steps skipped - nothing to do!');
+            return Command::SUCCESS;
+        }
+
+        foreach ($commands as $cmd) {
+            $io->section(sprintf('Running: %s', $cmd['description']));
+
+            try {
+                // Create process to run the command
+                $process = new Process([
+                    PHP_BINARY,
+                    'bin/console',
+                    $cmd['name'],
+                    '--no-interaction'
+                ]);
+                $process->setTimeout(600); // 10 minutes timeout
+
+                // Run and stream output in real-time
+                $process->run(function ($type, $buffer) use ($output) {
+                    $output->write($buffer);
+                });
+
+                if (!$process->isSuccessful()) {
+                    $io->error(sprintf(
+                        '%s failed with exit code: %d',
+                        $cmd['description'],
+                        $process->getExitCode()
+                    ));
+
+                    $errorOutput = $process->getErrorOutput();
+                    if ($errorOutput) {
+                        $io->text('Error output:');
+                        $io->text($errorOutput);
+                    }
+
+                    return Command::FAILURE;
+                }
+
+                $io->success(sprintf('✓ %s completed', $cmd['description']));
+                $io->newLine();
+
+            } catch (\Exception $e) {
+                $io->error(sprintf('Failed to run %s: %s', $cmd['name'], $e->getMessage()));
+                return Command::FAILURE;
+            }
+        }
+
+        $io->success('✓ All post-processing commands completed successfully!');
+
+        $io->newLine();
+        $io->text([
+            'Commands executed:',
+            sprintf('  • articles:qa: %s', $skipQa ? 'skipped' : 'completed'),
+            sprintf('  • articles:index: %s', $skipIndex ? 'skipped' : 'completed'),
+            sprintf('  • articles:indexed: %s', $skipIndexed ? 'skipped' : 'completed'),
+        ]);
+
+        return Command::SUCCESS;
+    }
+}
+

src/Controller/HighlightsController.php

@@ -6,6 +6,8 @@ namespace App\Controller;
 
 use App\Service\HighlightService;
 use App\Service\NostrClient;
+use App\Service\NostrLinkParser;
+use App\Service\RedisViewStore;
 use nostriphant\NIP19\Bech32;
 use Psr\Log\LoggerInterface;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
@@ -23,16 +25,58 @@ class HighlightsController extends AbstractController
         private readonly NostrClient $nostrClient,
         private readonly HighlightService $highlightService,
         private readonly LoggerInterface $logger,
-        private readonly \App\Service\NostrLinkParser $nostrLinkParser,
+        private readonly NostrLinkParser $nostrLinkParser,
+        private readonly RedisViewStore $viewStore,
     ) {}
 
     #[Route('/highlights', name: 'highlights')]
     public function index(CacheInterface $cache): Response
     {
         try {
+            // Fast path: Try Redis views first (single GET)
+            $cachedView = $this->viewStore->fetchLatestHighlights();
+
+            if ($cachedView !== null) {
+                // Use Redis view - extract highlights data
+                $highlights = [];
+
+                foreach ($cachedView as $baseObject) {
+                    if (isset($baseObject['highlight']) && isset($baseObject['article'])) {
+                        // Transform Redis view format to legacy highlight format
+                        $highlight = [
+                            'id' => $baseObject['highlight']['eventId'] ?? null,
+                            'content' => $baseObject['highlight']['content'] ?? '',
+                            'created_at' => isset($baseObject['highlight']['createdAt'])
+                                ? strtotime($baseObject['highlight']['createdAt'])
+                                : time(),
+                            'pubkey' => $baseObject['highlight']['pubkey'] ?? null,
+                            'context' => $baseObject['highlight']['context'] ?? null,
+                            'article_ref' => $baseObject['article']['eventId'] ?? null,
+                            'article_title' => $baseObject['article']['title'] ?? null,
+                            'article_author' => $baseObject['article']['pubkey'] ?? null,
+                            'article_slug' => $baseObject['article']['slug'] ?? null,
+                            'profile' => $baseObject['author'] ?? null, // Highlight author profile
+                            'article_author_profile' => $baseObject['profiles'][$baseObject['article']['pubkey']] ?? null,
+                        ];
+
+                        $highlights[] = $highlight;
+                    }
+                }
+
+                $this->logger->info('Loaded highlights from Redis view', ['count' => count($highlights)]);
+
+                return $this->render('pages/highlights.html.twig', [
+                    'highlights' => $highlights,
+                    'total' => count($highlights),
+                    'from_redis_view' => true,
+                ]);
+            }
+
+            // Fallback path: Use old cache system if Redis view not available
+            $this->logger->debug('Redis view not found, falling back to Nostr relay fetch');
+
             // Cache key for highlights
             $cacheKey = 'global_article_highlights';
-            $cache->delete($cacheKey);
             // Get highlights from cache or fetch fresh
             $highlights = $cache->get($cacheKey, function (ItemInterface $item) {
                 $item->expiresAfter(self::CACHE_TTL);
@@ -41,9 +85,6 @@ class HighlightsController extends AbstractController
                     // Fetch highlights that reference articles (kind 30023)
                     $events = $this->nostrClient->getArticleHighlights(self::MAX_DISPLAY_HIGHLIGHTS);
 
-                    // Save raw events to database first (group by article)
-                    //$this->saveHighlightsToDatabase($events);
-
                     // Process and enrich the highlights for display
                     return $this->processHighlights($events);
                 } catch (\Exception $e) {
@@ -57,6 +98,7 @@ class HighlightsController extends AbstractController
             return $this->render('pages/highlights.html.twig', [
                 'highlights' => $highlights,
                 'total' => count($highlights),
+                'from_redis_view' => false,
             ]);
 
         } catch (\Exception $e) {

src/Service/ArticleEventProjector.php

@@ -13,6 +13,8 @@ use Psr\Log\LoggerInterface;
  * Projects Nostr article events into the database
  * Handles the conversion from event format to Article entity and persistence
  * Also processes markdown content to HTML for performance optimization
+ *
+ * Note: Post-processing (QA, indexing) is handled by cron job running articles:post-process
  */
 class ArticleEventProjector
 {
@@ -21,7 +23,7 @@ class ArticleEventProjector
         private readonly EntityManagerInterface $entityManager,
         private readonly ManagerRegistry $managerRegistry,
         private readonly LoggerInterface $logger,
-        private readonly Converter $converter
+        private readonly Converter $converter,
     ) {
     }
 
@@ -82,6 +84,10 @@ class ArticleEventProjector
                     'event_id' => $article->getEventId(),
                     'db_id' => $article->getId()
                 ]);
+
+                // Note: Post-processing (QA, indexing) will be handled by cron job
+                // See: docker/cron/post_process_articles.sh (runs every 5 minutes)
+
             } else {
                 $this->logger->debug('Article already exists in database, skipping', [
                     'event_id' => $article->getEventId(),

templates/pages/article.html.twig

@@ -136,7 +136,7 @@
             <hr class="divider" />
             <div class="tags">
                 {% for tag in article.topics %}
-                    <a href="{{ path('forum_tag', {'tag': tag}) }}" class="tag">#{{ tag }}</a>
+                    <a href="{{ path('forum_tag', {'tag': tag|url_encode}) }}" class="tag">#{{ tag }}</a>
                 {% endfor %}
             </div>
             {% endif %}