From 2e6b7e178a22c2a63c91cfa5302f42b1efd33c65 Mon Sep 17 00:00:00 2001 From: Silberengel Date: Thu, 23 Apr 2026 10:00:48 +0200 Subject: [PATCH] auto-remove deletion events --- src/Command/PrewarmCommand.php | 59 ++++- src/Enum/KindsEnum.php | 1 + src/Repository/ArticleRepository.php | 5 + src/Service/MagazineIndexStore.php | 23 ++ src/Service/Nip09DeletionApplier.php | 359 +++++++++++++++++++++++++++ src/Service/NostrClient.php | 59 +++++ 6 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 src/Service/Nip09DeletionApplier.php diff --git a/src/Command/PrewarmCommand.php b/src/Command/PrewarmCommand.php index bc54a37..9dc6074 100644 --- a/src/Command/PrewarmCommand.php +++ b/src/Command/PrewarmCommand.php @@ -9,6 +9,7 @@ use App\Repository\ArticleRepository; use App\Service\ArticleCommentThreadLoader; use App\Service\CacheService; use App\Service\MagazineRefresher; +use App\Service\Nip09DeletionApplier; use App\Service\NostrClient; use Psr\Log\LoggerInterface; use swentel\nostr\Key\Key; @@ -26,12 +27,13 @@ use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface; */ #[AsCommand( name: 'app:prewarm', - description: 'Refresh magazine indices, profile metadata cache, and comment thread caches (use --no-comments to skip comments)', + description: 'Refresh magazine indices, NIP-09 deletions, profile metadata, and comment caches', )] final class PrewarmCommand extends Command { public function __construct( private readonly MagazineRefresher $magazineRefresher, + private readonly Nip09DeletionApplier $nip09DeletionApplier, private readonly CacheService $cacheService, private readonly NostrClient $nostrClient, private readonly ArticleRepository $articleRepository, @@ -46,6 +48,8 @@ final class PrewarmCommand extends Command { $this ->addOption('no-magazine', null, InputOption::VALUE_NONE, 'Skip magazine 30040 index fetch') + ->addOption('no-deletions', null, InputOption::VALUE_NONE, 'Skip NIP-09 kind 5 deletion sync (30023/30024 DB + 30040 magazine cache)') + ->addOption('deletion-since', null, InputOption::VALUE_REQUIRED, 'strtotime() window start for kind 5 fetch', '-2 month') ->addOption('no-metadata', null, InputOption::VALUE_NONE, 'Skip Nostr profile metadata cache') ->addOption('no-comments', null, InputOption::VALUE_NONE, 'Skip comment thread cache') ->addOption('magazine-budget', null, InputOption::VALUE_REQUIRED, 'Seconds wall time for magazine relay refresh', '30') @@ -79,6 +83,59 @@ final class PrewarmCommand extends Command // MagazineRefresher sets max_execution_time (e.g. 60 for budget 30); restore before metadata. $this->disableCliExecutionTimeLimit(); + if (!$input->getOption('no-deletions')) { + $io->section('NIP-09 deletions (kind 5 → 30023/30024 / 30040)'); + $sinceStr = (string) $input->getOption('deletion-since'); + $since = strtotime($sinceStr); + if ($since === false) { + $since = strtotime('-2 month'); + } + $until = time(); + $deletionPubkeys = []; + foreach ($this->articleRepository->findDistinctAuthorPubkeys() as $pk) { + if (\is_string($pk) && 64 === \strlen($pk)) { + $deletionPubkeys[] = $pk; + } + } + $npubParam = (string) $this->params->get('npub'); + if (str_starts_with($npubParam, 'npub')) { + try { + $sitePk = $keys->convertToHex($npubParam); + if ($sitePk !== '' && 64 === \strlen($sitePk) && !\in_array($sitePk, $deletionPubkeys, true)) { + $deletionPubkeys[] = $sitePk; + } + } catch (\Throwable) { + } + } + if ($deletionPubkeys === []) { + $io->note('No author pubkeys; skipping kind 5 deletion fetch.'); + } else { + try { + $kind5 = $this->nostrClient->fetchKind5DeletionEventsForAuthors( + $deletionPubkeys, + $since, + $until, + 40 + ); + $st = $this->nip09DeletionApplier->apply($kind5); + $io->writeln(sprintf( + 'Kind 5 events: %d (deduped). Articles removed: %d; magazine root/category cache entries removed: %d / %d.', + \count($kind5), + $st['articles_removed'], + $st['magazine_roots'], + $st['magazine_categories'] + )); + } catch (\Throwable $e) { + $this->logger->error('app:prewarm NIP-09 failed', ['exception' => $e]); + $io->warning('NIP-09 step failed: '.$e->getMessage()); + } + } + } else { + $io->note('Skipping NIP-09 deletions (--no-deletions).'); + } + + $this->disableCliExecutionTimeLimit(); + if (!$input->getOption('no-metadata')) { $io->section('Author metadata (cache)'); $pubkeys = $this->articleRepository->findDistinctAuthorPubkeys(); diff --git a/src/Enum/KindsEnum.php b/src/Enum/KindsEnum.php index 714f1fd..95f7295 100644 --- a/src/Enum/KindsEnum.php +++ b/src/Enum/KindsEnum.php @@ -5,6 +5,7 @@ namespace App\Enum; enum KindsEnum: int { case METADATA = 0; // metadata, NIP-01 + case DELETION_REQUEST = 5; // NIP-09 case TEXT_NOTE = 1; // text note, NIP-01, will not implement case FOLLOWS = 3; case REPOST = 6; // Only wraps kind 1, NIP-18, will not implement diff --git a/src/Repository/ArticleRepository.php b/src/Repository/ArticleRepository.php index 901ae83..bad8686 100644 --- a/src/Repository/ArticleRepository.php +++ b/src/Repository/ArticleRepository.php @@ -125,6 +125,11 @@ class ArticleRepository extends ServiceEntityRepository ->getSingleColumnResult(); } + public function findOneByEventId(string $eventId): ?Article + { + return $this->findOneBy(['eventId' => $eventId]); + } + /** * Find articles by author's public key */ diff --git a/src/Service/MagazineIndexStore.php b/src/Service/MagazineIndexStore.php index 8c6d1ed..755df7c 100644 --- a/src/Service/MagazineIndexStore.php +++ b/src/Service/MagazineIndexStore.php @@ -73,6 +73,29 @@ final class MagazineIndexStore $this->pool->save($item); } + /** + * Remove a cached category index (NIP-09 / local invalidation). + * + * @throws InvalidArgumentException + */ + public function deleteCategory(string $slug): void + { + if ($slug === '') { + return; + } + $this->pool->deleteItem(self::CAT_PREFIX.$slug); + } + + /** + * Remove the cached root magazine index for this npub + d_tag. + * + * @throws InvalidArgumentException + */ + public function deleteRoot(string $npub, string $dTag): void + { + $this->pool->deleteItem($this->rootKey($npub, $dTag)); + } + private function rootKey(string $npub, string $dTag): string { return self::ROOT_PREFIX.hash('sha256', $npub."\0".$dTag); diff --git a/src/Service/Nip09DeletionApplier.php b/src/Service/Nip09DeletionApplier.php new file mode 100644 index 0000000..49c5378 --- /dev/null +++ b/src/Service/Nip09DeletionApplier.php @@ -0,0 +1,359 @@ + $deletionEvents Kind-5 events from relays (e.g. {@see NostrClient::fetchKind5DeletionEventsForAuthors}) + * + * @return array{articles_removed: int, magazine_roots: int, magazine_categories: int} + */ + public function apply(array $deletionEvents): array + { + $articlesRemoved = 0; + $articlesPendingFlush = 0; + $roots = 0; + $cats = 0; + $seenArticleIds = []; + + foreach ($deletionEvents as $ev) { + if (!\is_object($ev)) { + continue; + } + if ((int) ($ev->kind ?? 0) !== KindsEnum::DELETION_REQUEST->value) { + continue; + } + $deletionPubkey = (string) ($ev->pubkey ?? ''); + if (64 !== \strlen($deletionPubkey)) { + continue; + } + + [$eIds, $eKinds] = $this->parseETags($ev); + $aAddrs = $this->parseATags($ev); + + foreach ($eIds as $i => $eId) { + if (64 !== \strlen($eId)) { + continue; + } + $declared = $eKinds[$i] ?? null; + if ($declared !== null + && !\in_array($declared, [30023, 30024, 30040, 1], true)) { + // Other kinds: we do not mirror in this app; skip. + continue; + } + if ($declared === 1) { + continue; + } + if ($this->removeArticleByEventIdIfValid($eId, $deletionPubkey, $declared, $seenArticleIds)) { + ++$articlesRemoved; + ++$articlesPendingFlush; + continue; + } + // No article row: 30040 index (or mis-tagged kind); only skip unrelated kinds. + if ($declared === null || \in_array($declared, [30023, 30024, 30040], true)) { + $mag = $this->tryRemoveMagazine30040ByEventId($eId, $deletionPubkey); + if ($mag === 1) { + ++$roots; + } elseif ($mag === 2) { + ++$cats; + } + } + } + + foreach ($aAddrs as $addr) { + $r = $this->removeByNip33Address($addr, $deletionPubkey, $seenArticleIds); + $articlesRemoved += $r['articles']; + $articlesPendingFlush += $r['articles']; + $roots += $r['roots']; + $cats += $r['cats']; + } + } + + if ($articlesPendingFlush > 0) { + try { + $this->entityManager->flush(); + } catch (\Throwable $e) { + $this->logger->error('Nip09DeletionApplier: flush failed', ['exception' => $e]); + } + } + + return [ + 'articles_removed' => $articlesRemoved, + 'magazine_roots' => $roots, + 'magazine_categories' => $cats, + ]; + } + + /** 0 = none, 1 = root cache, 2 = category cache */ + private function tryRemoveMagazine30040ByEventId(string $eventId, string $deletionPubkey): int + { + $npub = (string) $this->params->get('npub'); + $dTag = (string) $this->params->get('d_tag'); + if ($npub === '' || $dTag === '') { + return 0; + } + $root = $this->magazineIndexStore->getRoot($npub, $dTag); + if ($root === null) { + return 0; + } + if ($this->eventIdMatches($root, $eventId) && $this->pubkeyEquals($root->getPubkey(), $deletionPubkey)) { + $this->magazineIndexStore->deleteRoot($npub, $dTag); + $this->logger->notice('NIP-09: removed cached magazine root index', [ + 'event_id' => $eventId, + ]); + + return 1; + } + foreach ($this->categorySlugsFromRoot($root) as $slug) { + $cat = $this->magazineIndexStore->getCategory($slug); + if ($cat === null) { + continue; + } + if ($this->eventIdMatches($cat, $eventId) && $this->pubkeyEquals($cat->getPubkey(), $deletionPubkey)) { + $this->magazineIndexStore->deleteCategory($slug); + $this->logger->notice('NIP-09: removed cached magazine category index', [ + 'event_id' => $eventId, + 'slug' => $slug, + ]); + + return 2; + } + } + + return 0; + } + + private function eventIdMatches(MagazineNostrEvent $e, string $eventId): bool + { + $a = strtolower($e->getId()); + $b = strtolower($eventId); + + return $a === $b; + } + + private function pubkeyEquals(string $a, string $b): bool + { + if (64 !== \strlen($a) || 64 !== \strlen($b)) { + return $a === $b; + } + + return strtolower($a) === strtolower($b); + } + + /** + * @return list + */ + private function categorySlugsFromRoot(MagazineNostrEvent $root): array + { + $slugs = []; + foreach ($root->getTags() as $tag) { + if (($tag[0] ?? null) !== 'a' || !isset($tag[1])) { + continue; + } + $parts = explode(':', (string) $tag[1], 3); + if (\count($parts) < 3) { + continue; + } + $s = trim((string) end($parts)); + if ($s !== '' && !\in_array($s, $slugs, true)) { + $slugs[] = $s; + } + } + + return $slugs; + } + + /** + * @param array $seenArticleIds + */ + private function removeArticleByEventIdIfValid( + string $eId, + string $deletionPubkey, + ?int $declaredKind, + array &$seenArticleIds, + ): bool { + if (isset($seenArticleIds[$eId])) { + return false; + } + $article = $this->articleRepository->findOneByEventId($eId); + if ($article === null) { + return false; + } + if (!$this->pubkeyEquals($article->getPubkey() ?? '', $deletionPubkey)) { + $this->logger->debug('NIP-09: ignore e tag (pubkey mismatch)', [ + 'event_id' => $eId, + ]); + + return false; + } + $k = $article->getKind()?->value; + if ($declaredKind !== null && $k !== null && $declaredKind !== $k) { + return false; + } + if ($k !== null && !\in_array($k, [KindsEnum::LONGFORM->value, KindsEnum::LONGFORM_DRAFT->value], true)) { + return false; + } + $this->entityManager->remove($article); + $seenArticleIds[$eId] = true; + $this->logger->notice('NIP-09: removed article from database', [ + 'event_id' => $eId, + 'kind' => $k, + ]); + + return true; + } + + /** + * NIP-33: `kind:pubkeyhex:d-identifier` + * + * @param array $seenArticleIds + * + * @return array{articles: int, roots: int, cats: int} + */ + private function removeByNip33Address(string $addr, string $deletionPubkey, array &$seenArticleIds): array + { + $out = ['articles' => 0, 'roots' => 0, 'cats' => 0]; + $parts = explode(':', $addr, 3); + if (\count($parts) < 3) { + return $out; + } + $kind = (int) $parts[0]; + $pk = (string) $parts[1]; + $d = trim((string) $parts[2]); + if ($d === '' || !$this->pubkeyEquals($pk, $deletionPubkey)) { + return $out; + } + + if ($kind === KindsEnum::LONGFORM->value || $kind === KindsEnum::LONGFORM_DRAFT->value) { + $article = $this->articleRepository->findOneBy(['pubkey' => $pk, 'slug' => $d]); + if ($article !== null) { + $eid = (string) ($article->getEventId() ?? ''); + $dedupeKey = $eid !== '' ? $eid : 'ps:'.$pk."\0".$d; + if (!isset($seenArticleIds[$dedupeKey])) { + $this->entityManager->remove($article); + $seenArticleIds[$dedupeKey] = true; + ++$out['articles']; + $this->logger->notice('NIP-09: removed article (a tag)', [ + 'address' => $addr, + ]); + } + } + + return $out; + } + + if ($kind === KindsEnum::PUBLICATION_INDEX->value) { + $npub = (string) $this->params->get('npub'); + $siteD = (string) $this->params->get('d_tag'); + $siteHex = ''; + if (str_starts_with($npub, 'npub1')) { + try { + $h = (new Key())->convertToHex($npub); + if (64 === \strlen($h)) { + $siteHex = $h; + } + } catch (\Throwable) { + } + } + if ($npub !== '' && $siteD !== '' && $d === $siteD && $siteHex !== '' && $this->pubkeyEquals($pk, $siteHex)) { + $this->magazineIndexStore->deleteRoot($npub, $siteD); + ++$out['roots']; + $this->logger->notice('NIP-09: removed magazine root (a tag)', ['address' => $addr]); + } else { + // Category cache is keyed by `d` only; the same d string can appear for different + // authors' 30040 events. Only remove if the cached event was authored by this deletion. + $cachedCat = $this->magazineIndexStore->getCategory($d); + if ($cachedCat === null) { + $this->logger->debug('NIP-09: skip category delete (nothing cached for d)', [ + 'address' => $addr, + 'd' => $d, + ]); + } elseif (!$this->pubkeyEquals($cachedCat->getPubkey(), $deletionPubkey)) { + $this->logger->debug('NIP-09: skip category delete (cached index author != deletion author)', [ + 'address' => $addr, + 'd' => $d, + ]); + } else { + $this->magazineIndexStore->deleteCategory($d); + ++$out['cats']; + $this->logger->notice('NIP-09: removed magazine category (a tag)', [ + 'address' => $addr, + 'd' => $d, + ]); + } + } + } + + return $out; + } + + /** + * @return array{0: list, 1: list} e-ids and parallel k kinds (NIP-09 example order) + */ + private function parseETags(object $ev): array + { + $eIds = []; + $kinds = []; + foreach ($ev->tags ?? [] as $tag) { + if (!\is_array($tag) || !isset($tag[0], $tag[1])) { + continue; + } + if ($tag[0] === 'e') { + $eIds[] = (string) $tag[1]; + } + if ($tag[0] === 'k') { + $kinds[] = (int) $tag[1]; + } + } + $pairs = []; + for ($i = 0; $i < \count($eIds); ++$i) { + $pairs[] = $kinds[$i] ?? null; + } + + return [$eIds, $pairs]; + } + + /** + * @return list NIP-33 addresses + */ + private function parseATags(object $ev): array + { + $a = []; + foreach ($ev->tags ?? [] as $tag) { + if (!\is_array($tag) || ($tag[0] ?? null) !== 'a' || !isset($tag[1])) { + continue; + } + $a[] = (string) $tag[1]; + } + + return $a; + } +} diff --git a/src/Service/NostrClient.php b/src/Service/NostrClient.php index a752f83..96bfb06 100644 --- a/src/Service/NostrClient.php +++ b/src/Service/NostrClient.php @@ -316,6 +316,65 @@ class NostrClient return $byPub; } + /** + * NIP-09 kind 5 deletion requests in $since..$until (unix), batched by author pubkey (hex). + * + * @param list $authorPubkeyHex + * @return list Deduplicated by event `id` (highest {@see created_at} kept) + */ + public function fetchKind5DeletionEventsForAuthors( + array $authorPubkeyHex, + int $since, + int $until, + int $authorsPerRequest = 40, + ): array { + $authorPubkeyHex = \array_values(\array_unique(\array_filter( + $authorPubkeyHex, + static fn (mixed $h): bool => \is_string($h) && 64 === \strlen($h), + ))); + if ($authorPubkeyHex === [] || $since >= $until) { + return []; + } + $authorsPerRequest = max(1, min(100, $authorsPerRequest)); + $byId = []; + foreach (array_chunk($authorPubkeyHex, $authorsPerRequest) as $chunk) { + $request = $this->createNostrRequest( + kinds: [KindsEnum::DELETION_REQUEST], + filters: [ + 'authors' => $chunk, + 'since' => $since, + 'until' => $until, + ], + ); + $t0 = microtime(true); + $events = $this->processResponse( + $request->send(), + static fn (object $event) => $event, + ); + $this->logger->info('nostr.nip09.kind5_chunk', [ + 'authors' => \count($chunk), + 'raw_events' => \count($events), + 'ms' => (int) round((microtime(true) - $t0) * 1000), + ]); + foreach ($events as $ev) { + if (!\is_object($ev) || (int) ($ev->kind ?? 0) !== KindsEnum::DELETION_REQUEST->value) { + continue; + } + $id = (string) ($ev->id ?? ''); + if (64 !== \strlen($id)) { + continue; + } + $t = (int) ($ev->created_at ?? 0); + if (isset($byId[$id]) && $t <= (int) ($byId[$id]->created_at ?? 0)) { + continue; + } + $byId[$id] = $ev; + } + } + + return array_values($byId); + } + /** * @throws \Exception */