From e62e8604a49fdec1879a4250de736b4943e209c0 Mon Sep 17 00:00:00 2001 From: Silberengel Date: Thu, 23 Apr 2026 09:45:30 +0200 Subject: [PATCH] correct updates from relays --- .env.dist | 5 + Makefile | 7 + README.md | 170 ++++----- compose.override.yaml | 2 + compose.yaml | 6 +- composer.json | 2 +- composer.lock | 131 +++---- config/services.yaml | 4 + config/unfold.yaml | 5 + docker/cron/Dockerfile | 35 +- docker/cron/README.md | 95 +---- docker/cron/crontab | 2 +- docker/cron/entry-cron.sh | 6 + docker/cron/index_articles.sh | 5 - docker/cron/prewarm_cron.sh | 9 + frankenphp/docker-entrypoint.sh | 7 + scripts/docker-prewarm.sh | 22 ++ src/Command/PrewarmCommand.php | 193 ++++++++++ src/Repository/ArticleRepository.php | 16 + src/Service/ArticleCommentThreadLoader.php | 27 +- src/Service/CacheService.php | 78 +++- src/Service/MagazineRefresher.php | 16 +- src/Service/NostrClient.php | 419 ++++++++++++++++++--- 23 files changed, 908 insertions(+), 354 deletions(-) create mode 100644 Makefile create mode 100644 docker/cron/entry-cron.sh delete mode 100644 docker/cron/index_articles.sh create mode 100644 docker/cron/prewarm_cron.sh create mode 100755 scripts/docker-prewarm.sh create mode 100644 src/Command/PrewarmCommand.php diff --git a/.env.dist b/.env.dist index 9043aad..aad4d46 100644 --- a/.env.dist +++ b/.env.dist @@ -34,8 +34,13 @@ MYSQL_PASSWORD=password # Root password is only used for the bundled database service, see compose.yaml # skip it, if you use your own MYSQL_ROOT_PASSWORD=root_password +# Set to 1 in the php service environment to run `app:prewarm` once after migrations on container start +# (magazine + metadata + comment cache; not a substitute for `scripts/docker-prewarm.sh`, which also runs articles:get). +# PREWARM_ON_START=0 # Hub deploy: optional full image ref (default silberengel/unfold:latest in compose.hub.yaml). # UNFOLD_DOCKER_IMAGE=silberengel/unfold:1.0.0 +# Optional extra CLI args for the docker `cron` service (full `app:prewarm` every 10 min). Example: --metadata-limit=100 --no-magazine +# PREWARM_FLAGS= # compose.hub.yaml: default host port is 9080. Use 80 only if nothing else binds it. Loopback-only example: # HTTP_PUBLISH=127.0.0.1:9080 # HTTP_PUBLISH=80 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..dd695f5 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +# Default compose file from repo root; override with `make prewarm COMPOSE=...` if needed. +COMPOSE ?= docker compose + +.PHONY: prewarm +prewarm: + @chmod +x scripts/docker-prewarm.sh 2>/dev/null || true + @./scripts/docker-prewarm.sh diff --git a/README.md b/README.md index 97f98cb..2a6b808 100644 --- a/README.md +++ b/README.md @@ -1,128 +1,130 @@ -# Unfold +# Unfold: Imwald -Unfold is a customizable framework for your Nostr-based magazine. +

+ Imwald +

-(This is the **Imwald** edition of Unfold.) +A Symfony + FrankenPHP site that **reads Nostr long-form articles (kind 30023)** and related data from relays, stores articles in **MySQL**, and serves pages with Twig. **Comments and profile metadata** are **cache-backed** (not the full source of truth in the DB). -## Setup +--- -### Clone the repository +## Requirements -```bash -git clone https://github.com/decent-newsroom/unfold.git -cd unfold -``` - -### Create the .env file +| Requirement | Version / notes | +|------------|-----------------| +| PHP | **≥ 8.3.13** (see `composer.json`) | +| Docker | Optional; recommended for local dev and production images | +| Database | MySQL **8.0** (configurable) | -Copy the example file `.env.dist` and replace placeholders with your actual configuration. +--- -If you have your own MySQL database, comment out the database service in `compose.yaml` and skip root password in `.env`. -There are additional comments to that effect in the files. +## Local development (Docker) -### Configure `config/unfold.yaml` +1. **Env:** copy `.env.dist` to `.env` and adjust if needed (especially `APP_SECRET` outside dev). +2. **Start stack** -Before running the application, review and update `config/unfold.yaml` to match your desired magazine settings, theme, and external links. This file controls: -- Magazine name, short name, and description -- Theme and color settings -- Community articles feature -- External footer links -- Other project-specific configuration + ```bash + docker compose up -d + ``` -Edit the values in `config/unfold.yaml` as needed for your deployment. +3. **App URL (default):** [http://127.0.0.1:9080](http://127.0.0.1:9080) + Port comes from `HTTP_PORT` in `.env` and `compose.override.yaml` (loopback only). -### Customizing Theme and Icons +4. **First-time DB:** migrations run on **php** container start when `migrations/` contains PHP files (see `frankenphp/docker-entrypoint.sh`). -You can override the default theme and icons by adding your own files to `/assets/theme/local/`. To do this: -- Copy the structure and file names from `/assets/theme/default/`. -- Place your custom `theme.css` and icon files in your theme folder. -- Update your configuration in `config/unfold.yaml` to reference your custom theme if needed. +| Service | Role | +|--------|------| +| `php` | FrankenPHP + Caddy, Symfony app, console | +| `database` | MySQL; dev exposes `127.0.0.1:3307 → 3306` for local clients | +| `cron` | Runs full **`app:prewarm` every 10 minutes**; repo bind-mounted at `/var/www/html` (see `docker/cron/`) | -This allows you to easily switch or update the look and feel of your magazine without modifying the default assets. +--- +## Backfill articles + warm caches (recommended) -### Build the Docker containers +To **migrate**, **import articles from Nostr** for a time window, then **prewarm** magazine indices, author metadata, and comment caches: -For development: ```bash -docker compose build +make prewarm ``` -For production (using production overrides), set `APP_ENV=prod` in your `.env` file, set a strong **`APP_SECRET`**, and run: +| Step (script order) | Command / effect | +|---------------------|------------------| +| 1 | `docker compose up -d --wait` — starts **php**, **database**, and **cron** (the `cron` image runs a full `app:prewarm` on a 10 min schedule) | +| 2 | `doctrine:migrations:migrate` | +| 3 | `articles:get -- '-2 month' 'now'` — sync long-form into MySQL for that window | +| 4 | `app:prewarm` — magazine **30040**, **kind-0** profiles, **comment** cache (default **`--comments-max=20`**, newest by `createdAt`) | -```bash -docker compose -f compose.yaml -f compose.prod.yaml build -``` +`make prewarm` brings the stack (including `cron`) up so scheduled prewarm is active. **Optional** extra arguments for the **cron**-scheduled `app:prewarm` go in **`.env`** as **`PREWARM_FLAGS`** (same as you might pass to `php bin/console app:prewarm …`); Compose passes them into the `cron` container. Example: `PREWARM_FLAGS="--metadata-limit=50 --no-magazine"`. **Restart** the `cron` service after changing `PREWARM_FLAGS` so the container reloads the env. Hub / `compose.hub.yaml` has no `cron` service; use a host timer or `exec` if you need the same there. -`compose.override.yaml` is meant for local development. For production, always pass **both** compose files for `up` as well, otherwise Docker Compose still merges the dev override (FrankenPHP dev image, port `9080`, etc.): +--- -```bash -docker compose -f compose.yaml -f compose.prod.yaml up -d -``` - -The production compose file publishes the app on **host port `80`** → container `80` (FrankenPHP / Caddy). Put **TLS and your public hostname** (e.g. `https://blog.imwald.eu`) in front with Apache or nginx as a reverse proxy to `http://127.0.0.1:80` on that host (or another port if you change `compose.prod.yaml`). - -Set **`TRUSTED_PROXIES`** to the CIDR of your reverse proxy (defaults in `compose.prod.yaml` cover Docker and private nets; include the proxy’s address if it is elsewhere). In **`APP_ENV=prod`**, `config/packages/framework.yaml` enables **`trusted_proxies`** from that env var so Symfony trusts `X-Forwarded-Proto` / `X-Forwarded-For` from the proxy; adjust the value if generated URLs or secure cookies are wrong behind HTTPS. +## Console commands (overview) -### Docker Hub (pre-built image) +| Command | Purpose | +|---------|---------| +| `articles:get ` | Pull long-form articles from Nostr for the time range, persist to DB | +| `app:prewarm` | Magazine relay refresh + metadata cache + comment cache warm | +| `doctrine:migrations:migrate` | Apply SQL migrations | +| `user:elevate` | (If used) user elevation helper | -To build the production FrankenPHP image and push it (example registry: [`silberengel/unfold`](https://hub.docker.com/r/silberengel/unfold)): - -```bash -docker login -# If the server is linux/amd64 and your builder is ARM, set --platform (omit if arch matches). -docker build --platform linux/amd64 --target frankenphp_prod -t silberengel/unfold:latest . -docker push silberengel/unfold:latest -``` +`php bin/console list` and `… -h` for full options. -Tag a release when you want a pinned version: +### `app:prewarm` (notable options) -```bash -docker tag silberengel/unfold:latest silberengel/unfold:1.0.0 -docker push silberengel/unfold:1.0.0 -``` +| Option | Default | Meaning | +|--------|---------|--------| +| `--no-magazine` | off | Skip magazine 30040 index | +| `--no-metadata` | off | Skip Nostr kind-0 / profile cache | +| `--no-comments` | off | Skip comment thread cache | +| `--metadata-limit` | `0` (all authors) | Cap distinct author pubkeys | +| `--metadata-batch` | `50` | Pubkeys per batched Nostr `REQ` | +| `--comments-max` | `20` | Newest **N** articles (by `createdAt` **DESC**); `0` = all (still bounded by budget) | +| `--comments-budget` | `120` | Max wall seconds for the comments phase | +| `--magazine-budget` | `30` | Max wall seconds for magazine refresh | -**On the remote server** you only need `compose.hub.yaml`, a `.env` with at least **`APP_SECRET`** (and `MYSQL_*` / `MYSQL_ROOT_PASSWORD` if you use the bundled MySQL), and Docker Compose. Copy `compose.hub.yaml` from the repo (or clone once and take that file). The stack publishes the app on **host port `9080`** → container `80` by default (so **`:80` stays free** for Apache/nginx). Point your reverse proxy at `http://127.0.0.1:9080`. To bind only loopback, set **`HTTP_PUBLISH=127.0.0.1:9080`**; to use host port **80** instead, set **`HTTP_PUBLISH=80`**. Override the image with **`UNFOLD_DOCKER_IMAGE=myuser/unfold:1.0.0`** if you use another name or tag. +Prewarm clears the PHP **CLI** execution time limit for that run; relay work can be slow. -```bash -docker compose -f compose.hub.yaml pull -docker compose -f compose.hub.yaml up -d -docker compose -f compose.hub.yaml exec php php bin/console doctrine:migrations:migrate --no-interaction -``` +### `PREWARM_ON_START` (optional) -The production image must include **compiled asset mapper files** under `public/assets/` (the Docker build runs `asset-map:compile`). If you ever see JS modules blocked because the MIME type is `text/html`, the static files are missing: rebuild and push the image, or run once on the server: +| Variable | Set where | Effect | +|----------|------------|--------| +| `PREWARM_ON_START=1` | **Compose `environment` on the `php` service** (not only Symfony `.env` inside the container) | After DB is up and migrations run, executes **`app:prewarm` once** on start. **Does not** run `articles:get`. | -`docker compose -f compose.hub.yaml exec php php bin/console asset-map:compile --no-debug` +For a full **Nostr backfill** + one-shot prewarm, use **`make prewarm`** (or a host **cron** / **systemd** timer) instead of relying on **`PREWARM_ON_START` alone**. -The default `compose.hub.yaml` stack includes the **MySQL** service like the main compose file. If you use an external database, remove the `database` service and the `depends_on` block from `compose.hub.yaml`, and set **`DATABASE_URL`** in the `php` service `environment` to your connection string. +--- -**MySQL `1045 Access denied` for `unfold_user`:** The official MySQL image only applies **`MYSQL_USER` / `MYSQL_PASSWORD` / `MYSQL_ROOT_PASSWORD` on the first start** of an empty data volume. If you change passwords in `.env` later, the files inside the **`database_data` volume** still hold the old users. Either set **`.env`** back to the **original** passwords, or stop the stack and remove the named volume (e.g. `docker compose -f compose.hub.yaml down` then `docker volume rm unfold_database_data` — **this deletes all DB data**), then **`up -d`** again with the passwords you want and run **migrations** again. +## Configuration -The repo’s **`cron`** service still expects a local build and bind-mounted source; for Hub deploys, run **`articles:get`** (and any other jobs) from a host **cron** or **systemd timer** calling `docker compose -f compose.hub.yaml exec -T php php bin/console …`. +| What | File | +|------|------| +| Site title, `npub`, `d_tag`, **relays** (`default_relay`, `article_relays`, `profile_relays`), theme | `config/unfold.yaml` (imported as Symfony parameters) | +| `DATABASE_URL`, `APP_SECRET`, `HTTP_PORT`, `MYSQL_*`, optional **`PREWARM_FLAGS`** (for the Docker `cron` service) | `.env` / `.env.local` (see `.env.dist`) | +| Service wiring (e.g. cache, `NostrClient` args) | `config/services.yaml` | -### Start the Docker containers (development) +**Relays (short):** `default_relay` and `article_relays` drive article sync and many queries; `profile_relays` are used **first** for kind-0 / profile fetches, then the merged default + article set (see `NostrClient`). -```bash -docker compose up -d -``` +--- +## Production / Hub image -### Run Database Migrations +| Topic | Notes | +|-------|--------| +| `compose.hub.yaml` | Runs a **pulled** image (default `silberengel/unfold:latest`), no local PHP app build. Override with `UNFOLD_DOCKER_IMAGE`. | +| HTTP publish | `HTTP_PUBLISH` in `.env` (default **9080** → container **80**). Set `TRUSTED_PROXIES` behind a reverse proxy. | +| Secrets | Set `APP_SECRET` and DB credentials in **real** env; do not commit production secrets. | -Before fetching or displaying articles, make sure your database schema is up to date. Run: +File header in `compose.hub.yaml` lists pull, migrate, and optional build/push one-liners. -```bash -docker compose exec php php bin/console doctrine:migrations:migrate --no-interaction -``` +--- -If you use **`compose.hub.yaml`**, prefix commands with `docker compose -f compose.hub.yaml` (for example `docker compose -f compose.hub.yaml exec php php bin/console …`). +## License -### Fetching Articles +**MIT** — see [`LICENSE`](LICENSE). -To fetch articles from the default relay for the last two months, run: +--- -```bash -docker compose exec php php bin/console articles:get -- '-2 month' 'now' -``` +## Project links (example) -You can adjust the date range as needed. This command will import articles into the local database. +Configurable under `parameters.external_links` in `config/unfold.yaml` (e.g. Unfold on GitHub, Decent Newsroom). Adjust for your deployment. diff --git a/compose.override.yaml b/compose.override.yaml index 8d00ac0..74482a0 100644 --- a/compose.override.yaml +++ b/compose.override.yaml @@ -13,6 +13,8 @@ services: # from the bind-mount for better performance by enabling the next line: - /app/vendor environment: + # Set to 1 to run `app:prewarm` after migrations on php container start (see README). + # PREWARM_ON_START: "0" # develop: xdebug_info(), better stack traces, etc. Use debug,develop for step debugging (IDE). # See https://xdebug.org/docs/all_settings#mode XDEBUG_MODE: "${XDEBUG_MODE:-develop}" diff --git a/compose.yaml b/compose.yaml index a918d76..4c72530 100644 --- a/compose.yaml +++ b/compose.yaml @@ -44,8 +44,12 @@ services: context: ./docker/cron volumes: - .:/var/www/html + environment: + # Passed through from the host .env (Compose substitution); crond jobs read /run/cron-prewarm.env at runtime. + PREWARM_FLAGS: ${PREWARM_FLAGS:-} depends_on: - - php + database: + condition: service_healthy volumes: caddy_data: diff --git a/composer.json b/composer.json index 13d5660..924fc5b 100644 --- a/composer.json +++ b/composer.json @@ -22,7 +22,7 @@ "phpdocumentor/reflection-docblock": "^5.6", "phpstan/phpdoc-parser": "^2.0", "runtime/frankenphp-symfony": "^0.2.0", - "swentel/nostr-php": "^1.5", + "swentel/nostr-php": "^1.9.4", "symfony/asset": "7.1.*", "symfony/asset-mapper": "7.1.*", "symfony/console": "7.1.*", diff --git a/composer.lock b/composer.lock index fd206b3..25f73e0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "35730266c171fc5bbbe90ce2c0781509", + "content-hash": "bd231af493534154dcbc0c3f059f329e", "packages": [ { "name": "bitwasp/bech32", @@ -1403,6 +1403,64 @@ }, "time": "2025-01-24T11:45:48+00:00" }, + { + "name": "dsbaars/chacha20", + "version": "0.3.0", + "source": { + "type": "git", + "url": "https://github.com/dsbaars/PHP-ChaCha20.git", + "reference": "4edfef042c329313935a3dd516278e27c2939b5f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/dsbaars/PHP-ChaCha20/zipball/4edfef042c329313935a3dd516278e27c2939b5f", + "reference": "4edfef042c329313935a3dd516278e27c2939b5f", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-deprecation-rules": "^2.0", + "phpunit/phpunit": "~12.4" + }, + "type": "library", + "autoload": { + "files": [ + "lib/functions.php" + ], + "psr-4": { + "ChaCha20\\": "lib" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Leigh", + "homepage": "https://github.com/lt" + }, + { + "name": "Djuri Baars", + "homepage": "https://github.com/dsbaars" + } + ], + "description": "Pure PHP implementation of the ChaCha20 encryption algorithm.", + "homepage": "https://github.com/dsbaars/PHP-ChaCha20", + "keywords": [ + "cipher", + "encryption", + "security", + "stream" + ], + "support": { + "source": "https://github.com/dsbaars/PHP-ChaCha20/tree/0.3.0" + }, + "time": "2025-11-28T16:51:46+00:00" + }, { "name": "embed/embed", "version": "v4.4.17", @@ -2112,59 +2170,6 @@ ], "time": "2024-12-08T08:18:47+00:00" }, - { - "name": "leigh/chacha20", - "version": "0.2.0", - "source": { - "type": "git", - "url": "https://github.com/lt/PHP-ChaCha20.git", - "reference": "7aeffd53228be384b4a8986c9a8d9578acb171a4" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/lt/PHP-ChaCha20/zipball/7aeffd53228be384b4a8986c9a8d9578acb171a4", - "reference": "7aeffd53228be384b4a8986c9a8d9578acb171a4", - "shasum": "" - }, - "require": { - "php": ">=7.0" - }, - "require-dev": { - "phpunit/phpunit": "~5.0" - }, - "type": "library", - "autoload": { - "files": [ - "lib/functions.php" - ], - "psr-4": { - "ChaCha20\\": "lib" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Leigh", - "homepage": "https://github.com/lt" - } - ], - "description": "Pure PHP implementation of the ChaCha20 encryption algorithm.", - "homepage": "https://github.com/lt/PHP-ChaCha20", - "keywords": [ - "cipher", - "encryption", - "security", - "stream" - ], - "support": { - "issues": "https://github.com/lt/PHP-ChaCha20/issues", - "source": "https://github.com/lt/PHP-ChaCha20/tree/0.2.0" - }, - "time": "2016-01-14T11:24:17+00:00" - }, { "name": "masterminds/html5", "version": "2.10.0", @@ -3888,25 +3893,25 @@ }, { "name": "swentel/nostr-php", - "version": "1.9.2", + "version": "1.9.4", "source": { "type": "git", "url": "https://github.com/nostrver-se/nostr-php.git", - "reference": "8fb8337354b2e9d48a901276c7814d7fa7b25653" + "reference": "e502540ea811199443e1fffcbdaef9048940399c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nostrver-se/nostr-php/zipball/8fb8337354b2e9d48a901276c7814d7fa7b25653", - "reference": "8fb8337354b2e9d48a901276c7814d7fa7b25653", + "url": "https://api.github.com/repos/nostrver-se/nostr-php/zipball/e502540ea811199443e1fffcbdaef9048940399c", + "reference": "e502540ea811199443e1fffcbdaef9048940399c", "shasum": "" }, "require": { "bitwasp/bech32": "^0.0.1", + "dsbaars/chacha20": "^0.3.0", "ext-gmp": "*", "ext-xml": "*", - "leigh/chacha20": "^0.2.0", "paragonie/ecc": "^2.4", - "php": ">=8.1 <8.5", + "php": ">=8.2 <8.6", "phrity/websocket": "^3.0", "simplito/elliptic-php": "^1.0" }, @@ -3947,9 +3952,9 @@ "chat": "https://t.me/nostr_php", "issue": "https://github.com/swentel/nostr-php/issues", "issues": "https://github.com/nostrver-se/nostr-php/issues", - "source": "https://github.com/nostrver-se/nostr-php/tree/1.9.2" + "source": "https://github.com/nostrver-se/nostr-php/tree/1.9.4" }, - "time": "2025-06-04T14:51:06+00:00" + "time": "2026-02-03T11:13:00+00:00" }, { "name": "symfony/asset", @@ -11316,5 +11321,5 @@ "ext-openssl": "*" }, "platform-dev": {}, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.9.0" } diff --git a/config/services.yaml b/config/services.yaml index 831c07a..5678955 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -35,6 +35,7 @@ services: arguments: $defaultRelayUrl: '%default_relay%' $articleRelayUrls: '%article_relays%' + $profileRelayUrls: '%profile_relays%' App\Twig\FooterLinksExtension: arguments: $footerLinksPath: '%footer_links%' @@ -46,3 +47,6 @@ services: App\Service\MagazineRefresher: arguments: $appCache: '@cache.app' + App\Service\CacheService: + arguments: + $appCache: '@cache.app' diff --git a/config/unfold.yaml b/config/unfold.yaml index 627e4aa..e55630c 100644 --- a/config/unfold.yaml +++ b/config/unfold.yaml @@ -12,6 +12,11 @@ parameters: # Extra wss:// URLs for article sync (articles:get), comment threads (NIP-22 / getArticleDiscussion), # and any request that merges the default set with author-specific relays. default_relay is first; duplicates ignored. article_relays: ['wss://christpill.nostr1.com', 'wss://nostr.land', 'wss://nostr.wine', 'wss://nostr21.com', 'wss://nostr.sovbit.host'] + # Kind-0 / profile fetches (author metadata, prewarm). Tried first, then default + article_relays (deduped). + profile_relays: + - 'wss://relay.damus.io' + - 'wss://nos.lol' + - 'wss://profiles.nostr1.com' # Example: # article_relays: # - 'wss://nos.lol' diff --git a/docker/cron/Dockerfile b/docker/cron/Dockerfile index 5750236..49cbbca 100644 --- a/docker/cron/Dockerfile +++ b/docker/cron/Dockerfile @@ -1,30 +1,21 @@ -FROM php:8.2-cli +FROM php:8.3-cli -# Install cron and Redis PHP extension dependencies -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash \ cron \ - libzip-dev \ - libicu-dev \ - libpq-dev \ - libonig-dev + && rm -rf /var/lib/apt/lists/* +ADD https://github.com/mlocati/docker-php-extension-installer/releases/latest/download/install-php-extensions /usr/local/bin/ +RUN chmod +x /usr/local/bin/install-php-extensions \ + && install-php-extensions pdo_mysql intl opcache zip gmp -# Set working directory WORKDIR /var/www/html -# Install Symfony CLI tools (optional) -# RUN curl -sS https://get.symfony.com/cli/installer | bash +COPY crontab /etc/cron.d/unfold-prewarm +COPY prewarm_cron.sh /prewarm_cron.sh +COPY entry-cron.sh /entry-cron.sh -# Copy cron and script -COPY crontab /etc/cron.d/app-cron -COPY index_articles.sh /index_articles.sh +RUN chmod 0644 /etc/cron.d/unfold-prewarm \ + && chmod +x /prewarm_cron.sh /entry-cron.sh -# Set permissions -RUN chmod 0644 /etc/cron.d/app-cron && \ - chmod +x /index_articles.sh - -# Apply cron job -RUN crontab /etc/cron.d/app-cron - -# Run cron in the foreground -CMD ["cron", "-f"] +CMD ["/entry-cron.sh"] diff --git a/docker/cron/README.md b/docker/cron/README.md index ee65164..ba20824 100644 --- a/docker/cron/README.md +++ b/docker/cron/README.md @@ -1,94 +1,15 @@ +# `cron` service (Docker) -# 🕒 Cron Job Container +The `cron` image runs a single job: **`php bin/console app:prewarm` every 10 minutes**, against the app tree bind-mounted at `/var/www/html`. -This folder contains the Docker configuration to run scheduled Symfony commands via cron inside a separate container. +- **Flags:** set **`PREWARM_FLAGS`** in the project `.env` (Compose injects it). Example: `PREWARM_FLAGS="--metadata-limit=30 --no-magazine"`. After editing, run `docker compose up -d --force-recreate cron` (or `docker compose up -d cron`) so the container gets the new value. If unset, `app:prewarm` uses its **built-in defaults** (same idea as running the console with no args). -- Run Symfony console commands periodically using a cron schedule (e.g. every 6 hours) -- Decouple scheduled jobs from the main PHP/FPM container -- Easily manage and test cron execution in a Dockerized Symfony project +- **How env reaches cron:** the entrypoint writes `PREWARM_FLAGS` to `/run/cron-prewarm.env` at boot, because the system `crond` does not pass the container environment into crontab jobs. ---- +- **Logs inside the container:** `tail -f /var/log/cron.log` (e.g. `docker compose exec cron tail -f /var/log/cron.log`). -## Build & Run +- **PHP 8.3** extensions in the image are limited to what `app:prewarm` needs; the host **vendor** tree is what you mount from the repo. -1. **Build the cron image** - From the project root: - ```bash - docker-compose build cron - ``` +- **Not included in** `compose.hub.yaml` (no app source mount). For production images, use host **cron** / **systemd** to `exec` the same command. -2. **Start the cron container** - ```bash - docker-compose up -d cron - ``` - ---- - -## Cron Schedule - -The default cron schedule is set to run **every 6 hours**: - -```cron -0 */6 * * * root /run_commands.sh >> /var/log/cron.log 2>&1 -``` - -To customize the schedule, edit the `crontab` file and rebuild the container. - ---- - -## Testing & Debugging - -### Manually test the command runner - -You can run the script manually to check behavior without waiting for the cron trigger: - -```bash -docker-compose exec cron /run_commands.sh -``` - -### Check the cron output log - -```bash -docker-compose exec cron tail -f /var/log/cron.log -``` - -### Shell into the cron container - -```bash -docker-compose exec cron bash -``` - -Once inside, you can: -- Check crontab entries: `crontab -l` -- Manually trigger cron: `cron` or `cron -f` (in another session) - ---- - -## Customization - -- **Add/Remove Symfony Commands:** - Edit `run_commands.sh` to include the commands you want to run. - -- **Change Schedule:** - Edit `crontab` using standard cron syntax. - -- **Logging:** - Logs are sent to `/var/log/cron.log` inside the container. - ---- - -## Rebuilding After Changes - -If you modify the `crontab` or `run_commands.sh`, make sure to rebuild: - -```bash -docker-compose build cron -docker-compose up -d cron -``` - ---- - -## Notes - -- Symfony project source is mounted at `/var/www/html` via volume. -- Make sure your commands do **not rely on services** (like `php-fpm`) that are not running in this container. +Change the schedule: edit `docker/cron/crontab`, then `docker compose build cron && docker compose up -d cron`. diff --git a/docker/cron/crontab b/docker/cron/crontab index 3f37be3..f76824a 100644 --- a/docker/cron/crontab +++ b/docker/cron/crontab @@ -1 +1 @@ -*/5 * * * * /index_articles.sh >> /var/log/cron.log 2>&1 +*/10 * * * * root /prewarm_cron.sh >>/var/log/cron.log 2>&1 diff --git a/docker/cron/entry-cron.sh b/docker/cron/entry-cron.sh new file mode 100644 index 0000000..cbf8490 --- /dev/null +++ b/docker/cron/entry-cron.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail +# crond does not pass Compose env to job children; write once at boot for prewarm_cron.sh. +export PREWARM_FLAGS="${PREWARM_FLAGS:-}" +declare -p PREWARM_FLAGS >/run/cron-prewarm.env +exec cron -f diff --git a/docker/cron/index_articles.sh b/docker/cron/index_articles.sh deleted file mode 100644 index 831ee60..0000000 --- a/docker/cron/index_articles.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e -export PATH="/usr/local/bin:/usr/bin:/bin" - -php /var/www/html/bin/console articles:get -- '-6 min' 'now' diff --git a/docker/cron/prewarm_cron.sh b/docker/cron/prewarm_cron.sh new file mode 100644 index 0000000..530a2d7 --- /dev/null +++ b/docker/cron/prewarm_cron.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +# shellcheck source=/dev/null +if [[ -f /run/cron-prewarm.env ]]; then + source /run/cron-prewarm.env +fi +cd /var/www/html +# shellcheck disable=SC2086 +exec php bin/console app:prewarm ${PREWARM_FLAGS:-} diff --git a/frankenphp/docker-entrypoint.sh b/frankenphp/docker-entrypoint.sh index ff7df98..077ee8a 100644 --- a/frankenphp/docker-entrypoint.sh +++ b/frankenphp/docker-entrypoint.sh @@ -52,6 +52,13 @@ if [ "$1" = 'frankenphp' ] || [ "$1" = 'php' ] || [ "$1" = 'bin/console' ]; then if [ "$( find ./migrations -iname '*.php' -print -quit )" ]; then php bin/console doctrine:migrations:migrate --no-interaction --all-or-nothing fi + + # Optional: warm magazine index + metadata cache on container start (does not run articles:get). + # Prefer ./scripts/docker-prewarm.sh or `make prewarm` for full DB + relay backfill from the host. + if [ "${PREWARM_ON_START:-0}" = "1" ]; then + echo "PREWARM_ON_START=1: running app:prewarm..." + php bin/console app:prewarm || true + fi fi setfacl -R -m u:www-data:rwX -m u:"$(whoami)":rwX var diff --git a/scripts/docker-prewarm.sh b/scripts/docker-prewarm.sh new file mode 100755 index 0000000..272a1e0 --- /dev/null +++ b/scripts/docker-prewarm.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +# After `docker compose up`, run migrations, backfill long-form articles, and app:prewarm. +# Usage: from repository root: ./scripts/docker-prewarm.sh +# Or: make prewarm +set -eu + +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT_DIR" + +echo "==> docker compose up -d --wait (php, database, cron: full app:prewarm every 10 min; set PREWARM_FLAGS in .env to match your CLI)" +docker compose up -d --wait + +echo "==> doctrine:migrations:migrate" +docker compose exec -T php php bin/console doctrine:migrations:migrate --no-interaction + +echo "==> articles:get (last 2 months → now)" +docker compose exec -T php php bin/console articles:get -- '-2 month' 'now' + +echo "==> app:prewarm" +docker compose exec -T php php bin/console app:prewarm + +echo "Done." diff --git a/src/Command/PrewarmCommand.php b/src/Command/PrewarmCommand.php new file mode 100644 index 0000000..bc54a37 --- /dev/null +++ b/src/Command/PrewarmCommand.php @@ -0,0 +1,193 @@ +addOption('no-magazine', null, InputOption::VALUE_NONE, 'Skip magazine 30040 index fetch') + ->addOption('no-metadata', null, InputOption::VALUE_NONE, 'Skip Nostr profile metadata cache') + ->addOption('no-comments', null, InputOption::VALUE_NONE, 'Skip comment thread cache') + ->addOption('magazine-budget', null, InputOption::VALUE_REQUIRED, 'Seconds wall time for magazine relay refresh', '30') + ->addOption('metadata-limit', null, InputOption::VALUE_REQUIRED, 'Max distinct author pubkeys to warm (0 = all)', '0') + ->addOption('metadata-batch', null, InputOption::VALUE_REQUIRED, 'Kind-0 metadata: pubkeys per Nostr REQ (batched)', '50') + ->addOption('comments-max', null, InputOption::VALUE_REQUIRED, 'Newest N articles to warm comment cache for (0 = all, order: createdAt DESC)', '20') + ->addOption('comments-budget', null, InputOption::VALUE_REQUIRED, 'Max seconds for the whole comments phase', '120'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->disableCliExecutionTimeLimit(); + + $io = new SymfonyStyle($input, $output); + $keys = new Key(); + + if (!$input->getOption('no-magazine')) { + $budget = max(1, (int) $input->getOption('magazine-budget')); + $io->section('Magazine index (kinds 30040)'); + try { + $this->magazineRefresher->refreshFromRelays($budget, []); + $io->success('Magazine indices refreshed (within budget).'); + } catch (\Throwable $e) { + $this->logger->error('app:prewarm magazine failed', ['e' => $e]); + $io->warning('Magazine refresh failed: '.$e->getMessage()); + } + } else { + $io->note('Skipping magazine (--no-magazine).'); + } + + // MagazineRefresher sets max_execution_time (e.g. 60 for budget 30); restore before metadata. + $this->disableCliExecutionTimeLimit(); + + if (!$input->getOption('no-metadata')) { + $io->section('Author metadata (cache)'); + $pubkeys = $this->articleRepository->findDistinctAuthorPubkeys(); + $npubParam = (string) $this->params->get('npub'); + if (str_starts_with($npubParam, 'npub')) { + try { + $sitePk = $keys->convertToHex($npubParam); + if ($sitePk !== '' && !\in_array($sitePk, $pubkeys, true)) { + $pubkeys[] = $sitePk; + } + } catch (\Throwable) { + // ignore bad npub + } + } + $limit = (int) $input->getOption('metadata-limit'); + if ($limit > 0) { + $pubkeys = \array_slice($pubkeys, 0, $limit); + } + $toWarm = []; + foreach ($pubkeys as $pubkey) { + if (strlen($pubkey) === 64) { + $toWarm[] = $pubkey; + } + } + $total = \count($toWarm); + $n = 0; + if ($total === 0) { + $io->note('No valid author pubkeys to warm.'); + } else { + $batchSize = max(1, min(200, (int) $input->getOption('metadata-batch'))); + $io->writeln(sprintf( + 'Fetching kind-0 metadata: %d author(s) in Nostr requests of up to %d pubkeys each.', + $total, + $batchSize + )); + $bar = $io->createProgressBar($total); + $bar->start(); + try { + foreach (array_chunk($toWarm, $batchSize) as $chunk) { + $fetched = $this->nostrClient->fetchKind0MetadataForAuthors($chunk, $batchSize); + $n += $this->cacheService->putPrewarmMetadataBatch($chunk, $fetched, $keys); + $bar->advance(\count($chunk)); + } + } catch (\Throwable $e) { + $this->logger->error('app:prewarm metadata batch failed', ['exception' => $e]); + $io->error($e->getMessage()); + $bar->finish(); + $io->newLine(2); + + return Command::FAILURE; + } + $bar->finish(); + $io->newLine(2); + } + $io->success(sprintf('Warmed metadata for %d of %d author(s).', $n, $total)); + } else { + $io->note('Skipping metadata (--no-metadata).'); + } + + if ($input->getOption('no-comments')) { + $io->note('Skipping comments (--no-comments).'); + + return Command::SUCCESS; + } + + $maxArticles = (int) $input->getOption('comments-max'); + + $io->section('Comment / interaction cache'); + $deadline = microtime(true) + max(1, (int) $input->getOption('comments-budget')); + $qb = $this->articleRepository->createQueryBuilder('a') + ->where('a.slug IS NOT NULL') + ->andWhere("a.slug != ''") + ->andWhere('a.pubkey IS NOT NULL') + ->andWhere("a.pubkey != ''") + ->orderBy('a.createdAt', 'DESC'); + if ($maxArticles > 0) { + $qb->setMaxResults($maxArticles); + } + $articles = $qb->getQuery()->getResult(); + $w = 0; + /** @var Article $article */ + foreach ($articles as $article) { + if (microtime(true) >= $deadline) { + $io->warning('Comment phase stopped: comments-budget reached.'); + break; + } + $slug = trim((string) $article->getSlug()); + $pubkey = (string) $article->getPubkey(); + if ($slug === '' || strlen($pubkey) !== 64) { + continue; + } + $kind = $article->getKind()?->value ?? 30023; + $coordinate = $kind.':'.$pubkey.':'.$slug; + $eventHex = (string) ($article->getEventId() ?? ''); + try { + $this->commentThreadLoader->load($coordinate, $eventHex !== '' ? $eventHex : null); + ++$w; + } catch (\Throwable $e) { + $this->logger->warning('app:prewarm comment load', ['coord' => $coordinate, 'error' => $e->getMessage()]); + } + } + $io->success(sprintf('Warmed comment cache for %d of %d article(s).', $w, \count($articles))); + + return Command::SUCCESS; + } + + private function disableCliExecutionTimeLimit(): void + { + @set_time_limit(0); + @ini_set('max_execution_time', '0'); + } +} diff --git a/src/Repository/ArticleRepository.php b/src/Repository/ArticleRepository.php index f0ae417..901ae83 100644 --- a/src/Repository/ArticleRepository.php +++ b/src/Repository/ArticleRepository.php @@ -109,6 +109,22 @@ class ArticleRepository extends ServiceEntityRepository return $out; } + /** + * Distinct hex pubkeys for prewarming Nostr profile cache. + * + * @return list + */ + public function findDistinctAuthorPubkeys(): array + { + return $this->createQueryBuilder('a') + ->select('a.pubkey') + ->distinct() + ->where('a.pubkey IS NOT NULL') + ->andWhere("a.pubkey != ''") + ->getQuery() + ->getSingleColumnResult(); + } + /** * Find articles by author's public key */ diff --git a/src/Service/ArticleCommentThreadLoader.php b/src/Service/ArticleCommentThreadLoader.php index b023f57..529ddd6 100644 --- a/src/Service/ArticleCommentThreadLoader.php +++ b/src/Service/ArticleCommentThreadLoader.php @@ -47,27 +47,18 @@ final readonly class ArticleCommentThreadLoader 'elapsed_since_load_start_ms' => (int) round((microtime(true) - $t0) * 1000), ]); $tNostr = microtime(true); - try { - $out = $this->nostrClient->getArticleDiscussion($coordinate, $articleEventHexId); - $this->logger->info('comments.loader.nostr_ok', [ - 'nostr_elapsed_ms' => (int) round((microtime(true) - $tNostr) * 1000), - 'thread' => \count($out['thread'] ?? []), - 'quotes' => \count($out['quotes'] ?? []), - ]); - - return $out; - } catch (\Throwable $e) { - $this->logger->error('comments.loader.nostr_failed', [ - 'message' => $e->getMessage(), - 'exception_class' => \get_class($e), - 'nostr_elapsed_ms' => (int) round((microtime(true) - $tNostr) * 1000), - ]); + // On failure, let this throw: Symfony cache will not store a value, so a prior good thread is not replaced by []. + $out = $this->nostrClient->getArticleDiscussion($coordinate, $articleEventHexId); + $this->logger->info('comments.loader.nostr_ok', [ + 'nostr_elapsed_ms' => (int) round((microtime(true) - $tNostr) * 1000), + 'thread' => \count($out['thread'] ?? []), + 'quotes' => \count($out['quotes'] ?? []), + ]); - return ['thread' => [], 'quotes' => []]; - } + return $out; }); } catch (\Throwable $e) { - $this->logger->error('comments.loader.cache_failed', [ + $this->logger->error('comments.loader.cache_or_nostr_failed', [ 'message' => $e->getMessage(), 'exception_class' => \get_class($e), ]); diff --git a/src/Service/CacheService.php b/src/Service/CacheService.php index 346bd7c..522f823 100644 --- a/src/Service/CacheService.php +++ b/src/Service/CacheService.php @@ -2,20 +2,21 @@ namespace App\Service; +use Psr\Cache\CacheItemPoolInterface; use Psr\Cache\InvalidArgumentException; use Psr\Log\LoggerInterface; +use swentel\nostr\Key\Key; use Symfony\Contracts\Cache\CacheInterface; use Symfony\Contracts\Cache\ItemInterface; readonly class CacheService { - public function __construct( - private NostrClient $nostrClient, - private CacheInterface $cache, - private LoggerInterface $logger - ) - { + private NostrClient $nostrClient, + private CacheInterface $cache, + private LoggerInterface $logger, + private CacheItemPoolInterface $appCache, + ) { } /** @@ -26,25 +27,52 @@ readonly class CacheService { $cacheKey = '0_' . $npub; try { - return $this->cache->get($cacheKey, function (ItemInterface $item) use ($npub, $cacheKey) { + return $this->cache->get($cacheKey, function (ItemInterface $item) use ($npub) { $item->expiresAfter(3600); // 1 hour, adjust as needed try { $meta = $this->nostrClient->getNpubMetadata($npub); - $this->logger->info('Metadata:', ['meta' => json_encode($meta)]); + return json_decode($meta->content); } catch (\Exception $e) { - $this->logger->error('Error getting user data.', ['exception' => $e]); throw new MetadataRetrievalException('Failed to retrieve metadata', 0, $e); } }); } catch (\Exception|InvalidArgumentException $e) { - $this->logger->error('Error getting user data.', ['exception' => $e]); + $root = $e->getPrevious() ?? $e; + $this->logger->warning('Profile metadata fetch failed; using npub placeholder.', [ + 'npub' => $npub, + 'exception' => $root, + ]); $content = new \stdClass(); $content->name = substr($npub, 0, 8) . '…' . substr($npub, -4); + return $content; } } + /** + * @param list $authorPubkeyHex + * @param array $metadataByHex from {@see NostrClient::fetchKind0MetadataForAuthors} + */ + public function putPrewarmMetadataBatch(array $authorPubkeyHex, array $metadataByHex, Key $key): int + { + $n = 0; + foreach ($authorPubkeyHex as $hex) { + if (strlen($hex) !== 64) { + continue; + } + $npub = $key->convertPublicKeyToBech32($hex); + if (isset($metadataByHex[$hex]) && $metadataByHex[$hex] instanceof \stdClass) { + $this->putProfileInCache($npub, $metadataByHex[$hex]); + } else { + $this->putProfilePlaceholderInCache($npub); + } + ++$n; + } + + return $n; + } + public function getRelays($npub) { $cacheKey = '3_' . $npub; @@ -63,4 +91,34 @@ readonly class CacheService return []; } } + + private function putProfileInCache(string $npub, \stdClass $content): void + { + try { + $item = $this->appCache->getItem('0_'.$npub); + $item->set($content); + $item->expiresAfter(3600); + $this->appCache->save($item); + } catch (InvalidArgumentException $e) { + $this->logger->error('putProfileInCache', ['npub' => $npub, 'exception' => $e]); + } + } + + private function putProfilePlaceholderInCache(string $npub): void + { + try { + $item = $this->appCache->getItem('0_'.$npub); + if ($item->isHit()) { + // Prewarm miss: keep an earlier good (or any) value — do not downgrade to placeholder. + return; + } + } catch (InvalidArgumentException $e) { + $this->logger->error('putProfilePlaceholderInCache', ['npub' => $npub, 'exception' => $e]); + + return; + } + $c = new \stdClass(); + $c->name = substr($npub, 0, 8).'…'.substr($npub, -4); + $this->putProfileInCache($npub, $c); + } } diff --git a/src/Service/MagazineRefresher.php b/src/Service/MagazineRefresher.php index 2636603..fe8e60c 100644 --- a/src/Service/MagazineRefresher.php +++ b/src/Service/MagazineRefresher.php @@ -42,10 +42,17 @@ final class MagazineRefresher // (e.g. slow TLS) and can fatal. Cap once with headroom; the $deadline loop limits work. $this->applyExecutionTimeCap($budgetSeconds); + $defaultRelay = (string) $this->params->get('default_relay'); + $relayLabel = (string) (parse_url($defaultRelay, \PHP_URL_HOST) ?: $defaultRelay); + $root = $this->nostrClient->getMagazineIndex($npub, $dTag); if ($root === null) { - $this->logger->warning('MagazineRefresher: root index not returned from relay', [ + $this->logger->warning(sprintf( + 'MagazineRefresher: root index not returned (tried from %s)', + $relayLabel + ), [ 'd_tag' => $dTag, + 'relay' => $defaultRelay, ]); return; @@ -67,9 +74,14 @@ final class MagazineRefresher $this->store->putCategory($slug, $cat); } } catch (\Throwable $e) { - $this->logger->error('MagazineRefresher: category fetch failed', [ + $this->logger->error(sprintf( + 'MagazineRefresher: category fetch failed (relays from %s): %s', + $relayLabel, + $e->getMessage() + ), [ 'slug' => $slug, 'message' => $e->getMessage(), + 'relay' => $defaultRelay, ]); } } diff --git a/src/Service/NostrClient.php b/src/Service/NostrClient.php index 9be2131..a752f83 100644 --- a/src/Service/NostrClient.php +++ b/src/Service/NostrClient.php @@ -24,10 +24,14 @@ use Symfony\Contracts\Cache\ItemInterface; class NostrClient { + /** Per-relay WebSocket I/O cap (seconds), applied on each relay’s {@see \WebSocket\Client}. */ + private const RELAY_REQUEST_TIMEOUT_SEC = 15; + private RelaySet $defaultRelaySet; /** * @param list $articleRelayUrls extra relays for the default set (default_relay is always first) + * @param list $profileRelayUrls kind-0 / profile; merged for metadata (see {@see profileMetadataQueryRelayUrlList()}) */ public function __construct( private readonly EntityManagerInterface $entityManager, @@ -37,6 +41,7 @@ class NostrClient private readonly LoggerInterface $logger, private readonly string $defaultRelayUrl, private readonly array $articleRelayUrls, + private readonly array $profileRelayUrls, private readonly CacheInterface $relayQueryCache, ) { $this->defaultRelaySet = $this->buildArticleRelaySet(); @@ -89,6 +94,45 @@ class NostrClient return $rs; } + /** + * Host (or full URL) for log messages so console output shows which relay without opening context. + */ + private static function relayLogLabel(string $relayUrl): string + { + $host = parse_url($relayUrl, \PHP_URL_HOST); + if (\is_string($host) && $host !== '') { + return $host; + } + + return $relayUrl; + } + + private function newTimedRequest(RelaySet $relaySet, RequestMessage $requestMessage): Request + { + $request = new Request($relaySet, $requestMessage); + // 1.9.4+: Request::setTimeout() drives getResponseFromRelay(). Older: only WebSocket client on Relay. + if (method_exists($request, 'setTimeout')) { + $request->setTimeout(self::RELAY_REQUEST_TIMEOUT_SEC); + } else { + $this->applyRelaySocketTimeoutToSet($relaySet); + } + + return $request; + } + + /** + * Set per-relay WebSocket I/O cap. {@see RelaySet::send()} bypasses {@see Request}; use this there too. + */ + private function applyRelaySocketTimeoutToSet(RelaySet $relaySet): void + { + foreach ($relaySet->getRelays() as $relay) { + $client = $relay->getClient(); + if (method_exists($client, 'setTimeout')) { + $client->setTimeout(self::RELAY_REQUEST_TIMEOUT_SEC); + } + } + } + /** * Merges all configured article relays (default + article_relays) with the given URLs in order, deduped. * Used for comment threads (getArticleDiscussion), per-author fetches, etc. @@ -143,34 +187,165 @@ class NostrClient }); } + /** + * @return list Deduplicated profile relay URLs from config + */ + private function profileRelayUrlList(): array + { + $seen = []; + $out = []; + foreach ($this->profileRelayUrls as $url) { + if (!\is_string($url) || $url === '' || isset($seen[$url])) { + continue; + } + if (!str_starts_with($url, 'wss:')) { + continue; + } + $seen[$url] = true; + $out[] = $url; + } + + return $out; + } + + /** + * Profile (kind-0) queries: {@see profileRelayUrlList()} first (Damus, nos.lol, …), then default + article set. + * Order matters: {@see Request::send()} walks relays sequentially. + * + * @return list + */ + private function profileMetadataQueryRelayUrlList(): array + { + $seen = []; + $ordered = []; + foreach (array_merge($this->profileRelayUrlList(), $this->configuredArticleRelayUrlList()) as $u) { + if (!\is_string($u) || $u === '' || isset($seen[$u])) { + continue; + } + $seen[$u] = true; + $ordered[] = $u; + } + if ($ordered === []) { + $ordered[] = $this->defaultRelayUrl; + } + + return $ordered; + } + + /** + * Same relays for kind-0 metadata, without mutating {@see $this->defaultRelaySet}. + */ + private function relaySetForProfileMetadataFetch(): RelaySet + { + $relaySet = new RelaySet(); + foreach ($this->profileMetadataQueryRelayUrlList() as $url) { + $relaySet->addRelay(new Relay($url)); + } + + return $relaySet; + } + + /** + * Batched kind-0 profile fetch: one Nostr REQ per chunk with multiple "authors" (hex pubkeys). + * + * @param list $authorPubkeyHex + * @return array Newest kind-0 JSON per pubkey, keyed by hex + */ + public function fetchKind0MetadataForAuthors(array $authorPubkeyHex, int $authorsPerRequest = 50): array + { + $authorPubkeyHex = \array_values(\array_unique(\array_filter( + $authorPubkeyHex, + static fn (mixed $h): bool => \is_string($h) && 64 === \strlen($h), + ))); + if ($authorPubkeyHex === []) { + return []; + } + $authorsPerRequest = max(1, min(200, $authorsPerRequest)); + $byPub = []; + $relaysTried = $this->profileMetadataQueryRelayUrlList(); + $relaysTriedStr = implode(', ', array_map(self::relayLogLabel(...), $relaysTried)); + $relaySet = $this->relaySetForProfileMetadataFetch(); + $chunks = array_chunk($authorPubkeyHex, $authorsPerRequest); + foreach ($chunks as $i => $chunk) { + $t0 = microtime(true); + $request = $this->createNostrRequest( + kinds: [KindsEnum::METADATA], + filters: ['authors' => $chunk], + relaySet: $relaySet + ); + $events = $this->processResponse( + $request->send(), + static fn ($ev) => $ev, + ); + $this->logger->info('nostr.metadata.batch_chunk', [ + 'chunk' => 1 + $i, + 'of' => \count($chunks), + 'authors' => \count($chunk), + 'events' => \count($events), + 'relays' => $relaysTriedStr, + 'ms' => (int) round((microtime(true) - $t0) * 1000), + ]); + $newest = []; + foreach ($events as $ev) { + if (!\is_object($ev) || !isset($ev->pubkey, $ev->content)) { + continue; + } + $pk = (string) $ev->pubkey; + if (64 !== \strlen($pk)) { + continue; + } + $ts = (int) ($ev->created_at ?? 0); + if (isset($newest[$pk]) && $ts <= $newest[$pk]['t']) { + continue; + } + $newest[$pk] = ['ev' => $ev, 't' => $ts]; + } + foreach ($newest as $pk => $row) { + $ev = $row['ev']; + try { + $data = \json_decode((string) $ev->content, false, 512, \JSON_THROW_ON_ERROR); + } catch (\JsonException) { + continue; + } + if (\is_object($data)) { + $byPub[$pk] = $data; + } + } + } + + return $byPub; + } + /** * @throws \Exception */ public function getNpubMetadata($npub): \stdClass { - $relaySet = $this->defaultRelaySet; - $relaySet->addRelay(new Relay('wss://profiles.nostr1.com')); // profile aggregator - $this->logger->info('Getting metadata for npub', ['npub' => $npub]); - // Npubs are converted to hex for the request down the line + $relaysTried = $this->profileMetadataQueryRelayUrlList(); + $relaysTriedStr = implode(', ', array_map(self::relayLogLabel(...), $relaysTried)); + $relaySet = $this->relaySetForProfileMetadataFetch(); + $this->logger->info(sprintf('Getting metadata for npub (relays: %s)', $relaysTriedStr), ['npub' => $npub, 'relays' => $relaysTried]); $request = $this->createNostrRequest( kinds: [KindsEnum::METADATA], filters: ['authors' => [$npub]], relaySet: $relaySet ); - $events = $this->processResponse($request->send(), function($received) { - $this->logger->info('Getting metadata for npub', ['item' => $received]); - return $received; - }); + $events = $this->processResponse( + $request->send(), + function ($received) { + $this->logger->debug('nostr.metadata.relay_event', ['event' => $received]); - $this->logger->info('Getting metadata for npub', ['response' => $events]); + return $received; + }, + ); if (empty($events)) { - throw new \Exception('No metadata found for npub: ' . $npub); + throw new \Exception('No metadata for npub '.$npub.' (relays: '.$relaysTriedStr.')'); } - // Sort by date and return newest - usort($events, fn($a, $b) => $b->created_at <=> $a->created_at); + usort($events, static fn ($a, $b) => (int) ($b->created_at ?? 0) <=> (int) ($a->created_at ?? 0)); + return $events[0]; } @@ -197,7 +372,7 @@ class NostrClient } } - $request = new Request($relays, $requestMessage); + $request = $this->newTimedRequest($relays, $requestMessage); $wrappers = $this->processResponse($request->send(), function (object $event) { $w = new \stdClass(); @@ -220,6 +395,7 @@ class NostrClient $relaySet->addRelay($relay); } $relaySet->setMessage($eventMessage); + $this->applyRelaySocketTimeoutToSet($relaySet); // TODO handle responses appropriately return $relaySet->send(); } @@ -260,7 +436,7 @@ class NostrClient $filter->setUntil($until); $requestMessage = new RequestMessage($subscriptionId, [$filter]); - $request = new Request($this->defaultRelaySet, $requestMessage); + $request = $this->newTimedRequest($this->defaultRelaySet, $requestMessage); $wrappers = $this->processResponse($request->send(), function (object $event) { $w = new \stdClass(); @@ -281,9 +457,12 @@ class NostrClient if (empty($relayList)) { $topAuthorRelays = $this->getTopReputableRelaysForAuthor($author); $authorRelaySet = $this->createRelaySet($topAuthorRelays); + $relaysTried = $this->plannedRelayUrlsForSet($topAuthorRelays); } else { $authorRelaySet = $this->createRelaySet($relayList); + $relaysTried = $this->plannedRelayUrlsForSet($relayList); } + $relaysTriedStr = implode(', ', array_map(self::relayLogLabel(...), $relaysTried)); try { // Create request using the helper method for forest relay set @@ -310,8 +489,9 @@ class NostrClient $this->saveLongFormContent([$wrapper]); } } catch (\Exception $e) { - $this->logger->error('Error querying relays', [ - 'error' => $e->getMessage() + $this->logger->error(sprintf('Error querying relays (%s): %s', $relaysTriedStr, $e->getMessage()), [ + 'error' => $e->getMessage(), + 'relays' => $relaysTried, ]); throw new \Exception('Error querying relays', 0, $e); } @@ -501,7 +681,7 @@ class NostrClient $subscription = new Subscription(); $subscriptionId = $subscription->setId(); $requestMessage = new RequestMessage($subscriptionId, $filters); - $request = new Request($relaySet, $requestMessage); + $request = $this->newTimedRequest($relaySet, $requestMessage); $this->logger->info('nostr.article_discussion.req_sending', [ 'subscription_id' => $subscriptionId, @@ -521,13 +701,20 @@ class NostrClient ]); $this->logNostrWireResponseSummary('article_discussion', $response); } catch (\Throwable $e) { - $this->logger->error('nostr.article_discussion.req_send_failed', [ + $this->logger->error(sprintf( + 'nostr.article_discussion.req_send_failed (relays: %s): %s', + implode(', ', array_map(self::relayLogLabel(...), $plannedRelayUrls)), + $e->getMessage() + ), [ 'coordinate' => $coordinate, 'error' => $e->getMessage(), 'exception_class' => \get_class($e), + 'relays' => $plannedRelayUrls, ]); - return ['thread' => [], 'quotes' => []]; + // Do not return a successful empty shape: callers (e.g. comment cache) must not + // persist [] as if relays responded — that would clobber a previously good thread. + throw new \RuntimeException('Nostr request failed for article discussion', 0, $e); } $tParse = microtime(true); @@ -620,7 +807,11 @@ class NostrClient { foreach ($response as $relayUrl => $relayRes) { if ($relayRes instanceof \Throwable) { - $this->logger->warning('nostr.wire.relay_throwable', [ + $this->logger->warning(sprintf( + 'nostr.wire.relay_throwable [%s]: %s', + self::relayLogLabel($relayUrl), + $relayRes->getMessage() + ), [ 'context' => $context, 'relay' => $relayUrl, 'message' => $relayRes->getMessage(), @@ -630,7 +821,11 @@ class NostrClient continue; } if (!\is_iterable($relayRes)) { - $this->logger->warning('nostr.wire.relay_not_iterable', [ + $this->logger->warning(sprintf( + 'nostr.wire.relay_not_iterable [%s]: %s', + self::relayLogLabel($relayUrl), + \get_debug_type($relayRes) + ), [ 'context' => $context, 'relay' => $relayUrl, 'php_type' => \get_debug_type($relayRes), @@ -660,7 +855,7 @@ class NostrClient ++$counts['other']; } } - $this->logger->info('nostr.wire.relay_messages', [ + $this->logger->info(sprintf('nostr.wire.relay_messages [%s]', self::relayLogLabel($relayUrl)), [ 'context' => $context, 'relay' => $relayUrl, 'counts' => $counts, @@ -921,12 +1116,24 @@ class NostrClient $requestMessage = new RequestMessage($subscriptionId, [$filter]); try { - $request = new Request($this->defaultRelaySet, $requestMessage); + $request = $this->newTimedRequest($this->defaultRelaySet, $requestMessage); $response = $request->send(); $hasEvents = false; // Check if we got any events - foreach ($response as $value) { + foreach ($response as $relayUrl => $value) { + if ($value instanceof \Throwable) { + $this->logger->warning(sprintf( + '[%s] getArticles: %s', + self::relayLogLabel($relayUrl), + $value->getMessage() + ), ['relay' => $relayUrl]); + + continue; + } + if (!\is_iterable($value)) { + continue; + } foreach ($value as $item) { if ($item->type === 'EVENT') { if (!isset($articles[$item->event->id])) { @@ -941,31 +1148,58 @@ class NostrClient if (!$hasEvents && !empty($slugs)) { $this->logger->info('No results from theforest, trying default relays'); - $request = new Request($this->defaultRelaySet, $requestMessage); + $request = $this->newTimedRequest($this->defaultRelaySet, $requestMessage); $response = $request->send(); - foreach ($response as $value) { + foreach ($response as $relayUrl => $value) { + if ($value instanceof \Throwable) { + $this->logger->warning(sprintf( + '[%s] getArticles: %s', + self::relayLogLabel($relayUrl), + $value->getMessage() + ), ['relay' => $relayUrl]); + + continue; + } + if (!\is_iterable($value)) { + continue; + } foreach ($value as $item) { if ($item->type === 'EVENT') { if (!isset($articles[$item->event->id])) { $articles[$item->event->id] = $item->event; } - } elseif (in_array($item->type, ['AUTH', 'ERROR', 'NOTICE'])) { - $this->logger->error('An error while getting articles.', ['response' => $item]); + } elseif (in_array($item->type, ['AUTH', 'ERROR', 'NOTICE'], true)) { + $msg = (string) ($item->message ?? ''); + $this->logger->error(sprintf( + '[%s] %s while getting articles: %s', + self::relayLogLabel($relayUrl), + $item->type, + $msg !== '' ? $msg : '(no message)' + ), ['relay' => $relayUrl, 'response' => $item]); } } } } } catch (\Exception $e) { - $this->logger->error('Error querying relays', [ - 'error' => $e->getMessage() + $relaysTried = $this->configuredArticleRelayUrlList(); + $relaysStr = implode(', ', array_map(self::relayLogLabel(...), $relaysTried)); + $this->logger->error(sprintf('Error querying relays (%s): %s', $relaysStr, $e->getMessage()), [ + 'error' => $e->getMessage(), + 'relays' => $relaysTried, ]); // Fall back to default relay set - $request = new Request($this->defaultRelaySet, $requestMessage); + $request = $this->newTimedRequest($this->defaultRelaySet, $requestMessage); $response = $request->send(); - foreach ($response as $value) { + foreach ($response as $relayUrl => $value) { + if ($value instanceof \Throwable) { + continue; + } + if (!\is_iterable($value)) { + continue; + } foreach ($value as $item) { if ($item->type === 'EVENT') { if (!isset($articles[$item->event->id])) { @@ -1034,14 +1268,28 @@ class NostrClient $filter->setAuthors([$pubkey]); $filter->setTag('#d', [$slug]); $requestMessage = new RequestMessage($subscriptionId, [$filter]); + $relaysForLog = $this->plannedRelayUrlsForSet($relayList); + $relaysLogStr = implode(', ', array_map(self::relayLogLabel(...), $relaysForLog)); try { - $request = new Request($relaySet, $requestMessage); + $request = $this->newTimedRequest($relaySet, $requestMessage); $response = $request->send(); $found = false; // Check responses from each relay - foreach ($response as $value) { + foreach ($response as $relayUrl => $value) { + if ($value instanceof \Throwable) { + $this->logger->warning(sprintf( + '[%s] getArticlesByCoordinates: %s', + self::relayLogLabel($relayUrl), + $value->getMessage() + ), ['coordinate' => $coordinate, 'relay' => $relayUrl]); + + continue; + } + if (!\is_iterable($value)) { + continue; + } foreach ($value as $item) { if ($item->type === 'EVENT') { $articlesMap[$coordinate] = $item->event; @@ -1057,10 +1305,22 @@ class NostrClient 'coordinate' => $coordinate ]); - $request = new Request($this->defaultRelaySet, $requestMessage); + $request = $this->newTimedRequest($this->defaultRelaySet, $requestMessage); $response = $request->send(); - foreach ($response as $value) { + foreach ($response as $relayUrl => $value) { + if ($value instanceof \Throwable) { + $this->logger->warning(sprintf( + '[%s] getArticlesByCoordinates: %s', + self::relayLogLabel($relayUrl), + $value->getMessage() + ), ['coordinate' => $coordinate, 'relay' => $relayUrl]); + + continue; + } + if (!\is_iterable($value)) { + continue; + } foreach ($value as $item) { if ($item->type === 'EVENT') { $articlesMap[$coordinate] = $item->event; @@ -1070,9 +1330,14 @@ class NostrClient } } } catch (\Exception $e) { - $this->logger->error('Error fetching article', [ + $this->logger->error(sprintf( + 'Error fetching article (relays: %s): %s', + $relaysLogStr, + $e->getMessage() + ), [ 'coordinate' => $coordinate, - 'error' => $e->getMessage() + 'error' => $e->getMessage(), + 'relays' => $relaysForLog, ]); } } @@ -1102,24 +1367,27 @@ class NostrClient $requestMessage = new RequestMessage($subscriptionId, [$filter]); - return new Request($relaySet ?? $this->defaultRelaySet, $requestMessage); + return $this->newTimedRequest($relaySet ?? $this->defaultRelaySet, $requestMessage); } private function processResponse(array $response, callable $eventHandler): array { $results = []; foreach ($response as $relayUrl => $relayRes) { - // Skip if the relay response is an Exception - if ($relayRes instanceof \Exception) { - $this->logger->error('Relay error', [ + if ($relayRes instanceof \Throwable) { + $this->logger->error(sprintf( + 'Relay error at %s: %s', + self::relayLogLabel($relayUrl), + $relayRes->getMessage() + ), [ 'relay' => $relayUrl, - 'error' => $relayRes->getMessage() + 'error' => $relayRes->getMessage(), ]); continue; } $itemEstimate = \is_countable($relayRes) ? \count($relayRes) : null; - $this->logger->debug('Processing relay response', [ + $this->logger->debug(sprintf('Processing relay response from %s', self::relayLogLabel($relayUrl)), [ 'relay' => $relayUrl, 'item_count' => $itemEstimate, ]); @@ -1127,18 +1395,21 @@ class NostrClient foreach ($relayRes as $item) { try { if (!is_object($item)) { - $this->logger->warning('Invalid response item', [ + $this->logger->warning(sprintf( + 'Invalid response item from %s', + self::relayLogLabel($relayUrl) + ), [ 'relay' => $relayUrl, - 'item' => $item + 'item' => $item, ]); continue; } switch ($item->type) { case 'EVENT': - $this->logger->debug('Processing event', [ + $this->logger->debug(sprintf('Processing event from %s', self::relayLogLabel($relayUrl)), [ 'relay' => $relayUrl, - 'event_id' => $item->event->id ?? 'unknown' + 'event_id' => $item->event->id ?? 'unknown', ]); $result = $eventHandler($item->event); if ($result !== null) { @@ -1146,24 +1417,37 @@ class NostrClient } break; case 'AUTH': - $this->logger->warning('Relay requires authentication', [ + $this->logger->warning(sprintf( + 'Relay %s requires authentication', + self::relayLogLabel($relayUrl) + ), [ 'relay' => $relayUrl, - 'response' => $item + 'response' => $item, ]); break; case 'ERROR': case 'NOTICE': - $this->logger->warning('Relay error/notice', [ + $msg = (string) ($item->message ?? 'No message'); + $this->logger->warning(sprintf( + '[%s] %s: %s', + self::relayLogLabel($relayUrl), + $item->type, + $msg + ), [ 'relay' => $relayUrl, 'type' => $item->type, - 'message' => $item->message ?? 'No message' + 'message' => $msg, ]); break; } } catch (\Exception $e) { - $this->logger->error('Error processing event from relay', [ + $this->logger->error(sprintf( + 'Error processing event from relay %s: %s', + self::relayLogLabel($relayUrl), + $e->getMessage() + ), [ 'relay' => $relayUrl, - 'error' => $e->getMessage() + 'error' => $e->getMessage(), ]); continue; // Skip this item but continue processing others } @@ -1299,32 +1583,42 @@ class NostrClient */ public function getMagazineIndex(mixed $npub, mixed $dTag): ?PublicationEventEntity { - $entity = $this->queryMagazineIndex($npub, $dTag, $this->buildSingleRelaySet($this->defaultRelayUrl)); + $entity = $this->queryMagazineIndex( + $npub, + $dTag, + $this->buildSingleRelaySet($this->defaultRelayUrl), + self::relayLogLabel($this->defaultRelayUrl) + ); if ($entity !== null) { return $entity; } if (\count($this->configuredArticleRelayUrlList()) <= 1) { - $this->logger->warning('No magazine index found', ['npub' => $npub, 'dTag' => $dTag]); + $this->logger->warning(sprintf( + 'No magazine index found (tried %s)', + self::relayLogLabel($this->defaultRelayUrl) + ), ['npub' => $npub, 'dTag' => $dTag, 'relay' => $this->defaultRelayUrl]); return null; } $this->logger->notice('Magazine index not on default relay, falling back to full relay set', [ 'dTag' => $dTag, ]); + $fullListStr = implode(', ', array_map(self::relayLogLabel(...), $this->configuredArticleRelayUrlList())); - return $this->queryMagazineIndex($npub, $dTag, $this->defaultRelaySet); + return $this->queryMagazineIndex($npub, $dTag, $this->defaultRelaySet, $fullListStr); } - private function queryMagazineIndex(mixed $npub, mixed $dTag, RelaySet $relaySet): ?PublicationEventEntity + private function queryMagazineIndex(mixed $npub, mixed $dTag, RelaySet $relaySet, string $relaysForLog): ?PublicationEventEntity { $request = $this->createNostrRequest( [KindsEnum::PUBLICATION_INDEX], ['authors' => [(string) $npub], 'tag' => ['#d', [(string) $dTag]]], $relaySet, ); - $this->logger->info('Magazine index query', [ + $this->logger->info(sprintf('Magazine index query (relays: %s)', $relaysForLog), [ 'npub' => $npub, 'dTag' => $dTag, + 'relays' => $relaysForLog, ]); $response = $request->send(); $events = $this->processResponse($response, function ($received) { @@ -1392,9 +1686,14 @@ class NostrClient return null; }); } catch (\Throwable $e) { - $this->logger->error('ingestMissingLongformForCategoryCoordinates', [ + $this->logger->error(sprintf( + 'ingestMissingLongformForCategoryCoordinates [%s]: %s', + self::relayLogLabel($this->defaultRelayUrl), + $e->getMessage() + ), [ 'message' => $e->getMessage(), 'pubkey' => $g['pubkey'] ?? null, + 'relay' => $this->defaultRelayUrl, ]); } }