From 129fc4964ba736550cac9557dc1d103579f31f53 Mon Sep 17 00:00:00 2001 From: Silberengel Date: Wed, 22 Apr 2026 15:00:03 +0200 Subject: [PATCH] speed up the app some more --- src/Controller/MagazineSyncController.php | 5 +- src/Repository/ArticleRepository.php | 40 +++++++++ src/Service/MagazineContentService.php | 62 ++++++++----- src/Service/NostrClient.php | 105 ++++++++++++++++++++-- 4 files changed, 178 insertions(+), 34 deletions(-) diff --git a/src/Controller/MagazineSyncController.php b/src/Controller/MagazineSyncController.php index b8272aa..835492a 100644 --- a/src/Controller/MagazineSyncController.php +++ b/src/Controller/MagazineSyncController.php @@ -31,9 +31,6 @@ final class MagazineSyncController #[Route('/ux/magazine-sync', name: 'ux_magazine_sync', methods: ['GET'])] public function __invoke(Request $request): JsonResponse { - @set_time_limit(8); - @ini_set('max_execution_time', '8'); - try { $page = (string) $request->query->get('page', 'article'); if (!\in_array($page, ['home', 'category', 'article', 'articles'], true)) { @@ -44,7 +41,7 @@ final class MagazineSyncController $prefer = $slug !== '' ? [$slug] : []; try { - $this->refresher->refreshFromRelays(8, $prefer); + $this->refresher->refreshFromRelays(20, $prefer); } catch (\Throwable $e) { $this->logger->warning('MagazineSyncController: refresh failed', [ 'message' => $e->getMessage(), diff --git a/src/Repository/ArticleRepository.php b/src/Repository/ArticleRepository.php index 655b981..f0ae417 100644 --- a/src/Repository/ArticleRepository.php +++ b/src/Repository/ArticleRepository.php @@ -69,6 +69,46 @@ class ArticleRepository extends ServiceEntityRepository ->getResult(); } + /** + * Resolve NIP-33 `a` tags (kind:pubkey:identifier) to articles without conflating the same + * #d value across different authors. + * + * @param list $pairs + * @return array key "pubkey\0slug" (lowercase hex pubkey, trimmed slug) + */ + public function findByAuthorAndSlugIndexed(array $pairs): array + { + $pairs = array_values(array_filter($pairs, static fn (array $p): bool => $p['pubkey'] !== '' && $p['slug'] !== '')); + if ($pairs === []) { + return []; + } + + $qb = $this->createQueryBuilder('a'); + $orX = $qb->expr()->orX(); + foreach ($pairs as $i => $p) { + $orX->add($qb->expr()->andX( + $qb->expr()->eq('a.pubkey', ':pk'.$i), + $qb->expr()->eq('a.slug', ':sl'.$i) + )); + $qb->setParameter('pk'.$i, $p['pubkey']); + $qb->setParameter('sl'.$i, $p['slug']); + } + $qb->where($orX); + + /** @var list
$rows */ + $rows = $qb->getQuery()->getResult(); + $out = []; + foreach ($rows as $a) { + $pk = (string) $a->getPubkey(); + $sl = trim((string) $a->getSlug()); + if ($sl !== '') { + $out[$pk."\0".$sl] = $a; + } + } + + return $out; + } + /** * Find articles by author's public key */ diff --git a/src/Service/MagazineContentService.php b/src/Service/MagazineContentService.php index 566ab74..9ccb0ef 100644 --- a/src/Service/MagazineContentService.php +++ b/src/Service/MagazineContentService.php @@ -24,6 +24,7 @@ final class MagazineContentService private readonly MagazineRefresher $refresher, private readonly ParameterBagInterface $params, private readonly ArticleRepository $articleRepository, + private readonly NostrClient $nostrClient, ) { } @@ -37,9 +38,9 @@ final class MagazineContentService $npub = (string) $this->params->get('npub'); $dTag = (string) $this->params->get('d_tag'); if ($this->store->getRoot($npub, $dTag) === null) { - $this->refresher->refreshFromRelays(8, []); + $this->refresher->refreshFromRelays(20, []); } elseif ($this->shouldRevalidateRootFromRelay()) { - $this->refresher->refreshFromRelays(8, []); + $this->refresher->refreshFromRelays(20, []); } return $this->getHomeCategoryAIndexTagsFromStoreOnly(); @@ -101,7 +102,7 @@ final class MagazineContentService { $catIndex = $this->store->getCategory($slug); if ($catIndex === null) { - $this->refresher->refreshFromRelays(8, [$slug]); + $this->refresher->refreshFromRelays(20, [$slug]); $catIndex = $this->store->getCategory($slug); } $list = []; @@ -122,32 +123,45 @@ final class MagazineContentService } if (!empty($coordinates)) { - $slugs = array_map(static function ($coordinate) { + $pairs = []; + foreach ($coordinates as $coordinate) { $parts = explode(':', (string) $coordinate, 3); - - return trim((string) end($parts)); - }, $coordinates); - $slugs = array_values(array_filter($slugs, static fn (string $s): bool => $s !== '')); - $articles = $this->articleRepository->findBySlugsCriteria($slugs); - $slugMap = []; - foreach ($articles as $item) { - $s = trim((string) $item->getSlug()); - if ($s !== '') { - if (!isset($slugMap[$s])) { - $slugMap[$s] = $item; - } else { - $existingItem = $slugMap[$s]; - if ($item->getCreatedAt() > $existingItem->getCreatedAt()) { - $slugMap[$s] = $item; - } - } + if (\count($parts) < 3) { + continue; + } + $slugPart = trim((string) $parts[2]); + if ($slugPart === '') { + continue; + } + $pairs[] = [ + 'pubkey' => (string) $parts[1], + 'slug' => $slugPart, + ]; + } + $byAddress = $this->articleRepository->findByAuthorAndSlugIndexed($pairs); + $missing = []; + foreach ($coordinates as $coordinate) { + $parts = explode(':', (string) $coordinate, 3); + if (\count($parts) < 3) { + continue; + } + $k = (string) $parts[1]."\0".trim((string) $parts[2]); + if (!isset($byAddress[$k])) { + $missing[] = (string) $coordinate; } } + if ($missing !== []) { + $this->nostrClient->ingestMissingLongformForCategoryCoordinates($missing); + $byAddress = $this->articleRepository->findByAuthorAndSlugIndexed($pairs); + } foreach ($coordinates as $coordinate) { $parts = explode(':', (string) $coordinate, 3); - $slugKey = trim((string) end($parts)); - if ($slugKey !== '' && isset($slugMap[$slugKey])) { - $list[] = $slugMap[$slugKey]; + if (\count($parts) < 3) { + continue; + } + $k = (string) $parts[1]."\0".trim((string) $parts[2]); + if (isset($byAddress[$k])) { + $list[] = $byAddress[$k]; } } } diff --git a/src/Service/NostrClient.php b/src/Service/NostrClient.php index 0884f50..9be2131 100644 --- a/src/Service/NostrClient.php +++ b/src/Service/NostrClient.php @@ -76,6 +76,19 @@ class NostrClient return $relaySet; } + /** + * One relay for magazine 30040 lookups. {@see Request::send()} iterates every relay in the set + * sequentially; the full default set (5–6 wss) multiplies wall time — often 10s+ while a single + * relay returns in under 2s for the same filter. + */ + private function buildSingleRelaySet(string $wssUrl): RelaySet + { + $rs = new RelaySet(); + $rs->addRelay(new Relay($wssUrl)); + + return $rs; + } + /** * Merges all configured article relays (default + article_relays) with the given URLs in order, deduped. * Used for comment threads (getArticleDiscussion), per-author fetches, etc. @@ -1285,19 +1298,39 @@ class NostrClient * further nested 30040 indices. */ public function getMagazineIndex(mixed $npub, mixed $dTag): ?PublicationEventEntity + { + $entity = $this->queryMagazineIndex($npub, $dTag, $this->buildSingleRelaySet($this->defaultRelayUrl)); + if ($entity !== null) { + return $entity; + } + if (\count($this->configuredArticleRelayUrlList()) <= 1) { + $this->logger->warning('No magazine index found', ['npub' => $npub, 'dTag' => $dTag]); + + return null; + } + $this->logger->notice('Magazine index not on default relay, falling back to full relay set', [ + 'dTag' => $dTag, + ]); + + return $this->queryMagazineIndex($npub, $dTag, $this->defaultRelaySet); + } + + private function queryMagazineIndex(mixed $npub, mixed $dTag, RelaySet $relaySet): ?PublicationEventEntity { $request = $this->createNostrRequest( - kinds: [KindsEnum::PUBLICATION_INDEX], - filters: ['authors' => [(string) $npub], 'tag' => ['#d', [(string) $dTag]]], + [KindsEnum::PUBLICATION_INDEX], + ['authors' => [(string) $npub], 'tag' => ['#d', [(string) $dTag]]], + $relaySet, ); + $this->logger->info('Magazine index query', [ + 'npub' => $npub, + 'dTag' => $dTag, + ]); $response = $request->send(); - $this->logger->info('Getting magazine index', ['npub' => $npub, 'dTag' => $dTag, 'response' => $response]); - $events = $this->processResponse($response, function($received) { - $this->logger->info('Received magazine index event', ['item' => $received]); + $events = $this->processResponse($response, function ($received) { return $received; }); if (empty($events)) { - $this->logger->warning('No magazine index found', ['npub' => $npub, 'dTag' => $dTag]); return null; } usort($events, static function ($a, $b): int { @@ -1307,6 +1340,66 @@ class NostrClient return self::magazineEventToPublicationEntity($events[0]); } + /** + * Batch-fetch longform for category `a` coordinates that are not in the DB; one Nostr call per + * (author × kind) group, only the default relay (see {@see getMagazineIndex} rationale). + * + * @param list $addresses kind:pubkey:identifier + */ + public function ingestMissingLongformForCategoryCoordinates(array $addresses): void + { + if ($addresses === []) { + return; + } + $groups = []; + foreach ($addresses as $c) { + $parts = explode(':', (string) $c, 3); + if (\count($parts) < 3) { + continue; + } + $kind = (int) $parts[0]; + $pubkey = $parts[1]; + $d = trim((string) $parts[2]); + if ($d === '' || $kind <= 0) { + continue; + } + $gkey = $pubkey.':'.(string) $kind; + $groups[$gkey]['pubkey'] = $pubkey; + $groups[$gkey]['kind'] = $kind; + $groups[$gkey]['dTags'][] = $d; + } + foreach ($groups as $g) { + $dTags = array_values(array_unique($g['dTags'] ?? [])); + if ($dTags === [] || !isset($g['pubkey'], $g['kind'])) { + continue; + } + $kindEnum = KindsEnum::tryFrom((int) $g['kind']); + if ($kindEnum === null) { + $this->logger->notice('Skipping category coordinate with unknown kind', ['kind' => $g['kind']]); + + continue; + } + $request = $this->createNostrRequest( + [$kindEnum], + ['authors' => [(string) $g['pubkey']], 'tag' => ['#d', $dTags]], + $this->buildSingleRelaySet($this->defaultRelayUrl), + ); + try { + $this->processResponse($request->send(), function ($event) { + $article = $this->articleFactory->createFromLongFormContentEvent($event); + $this->saveEachArticleToTheDatabase($article); + + return null; + }); + } catch (\Throwable $e) { + $this->logger->error('ingestMissingLongformForCategoryCoordinates', [ + 'message' => $e->getMessage(), + 'pubkey' => $g['pubkey'] ?? null, + ]); + } + } + } + private static function magazineEventCreatedAt(mixed $event): int { if ($event instanceof PublicationEventEntity) {