diff --git a/Dockerfile b/Dockerfile index ab80f11..56a43e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,7 +95,7 @@ COPY --link . ./ RUN rm -Rf frankenphp/ RUN set -eux; \ - composer install --no-cache --prefer-dist --no-dev --no-progress + composer install --no-cache --prefer-dist --no-progress RUN set -eux; \ mkdir -p var/cache var/log; \ diff --git a/bin/relay/ingest.sh b/bin/relay/ingest.sh deleted file mode 100644 index 24a650d..0000000 --- a/bin/relay/ingest.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "[$(date)] Starting relay ingest..." - -# Config via env or defaults -UPSTREAMS=${UPSTREAMS:-"wss://relay.snort.social wss://relay.damus.io wss://relay.nostr.band"} -DAYS_ARTICLES=${DAYS_ARTICLES:-7} -DAYS_THREADS=${DAYS_THREADS:-3} - -# These two should be programmatically generated from app DB; allow overrides: -ARTICLE_E_LIST=${ARTICLE_E_LIST:-'[]'} # e.g. ["",""] -ARTICLE_A_LIST=${ARTICLE_A_LIST:-'[]'} # e.g. ["30023::",...] - -# Helper functions for date calculation -now_ts() { date +%s; } -since_days() { - local days=$1 - if command -v date >/dev/null 2>&1; then - # Try GNU date - if date -d "-${days} days" +%s 2>/dev/null; then - return 0 - # Try BSD date (macOS) - elif date -v-${days}d +%s 2>/dev/null; then - return 0 - fi - fi - # Fallback: rough calculation - echo $(( $(date +%s) - (days * 86400) )) -} - -# Build filters using jq if available, otherwise use basic JSON -if command -v jq >/dev/null 2>&1; then - FILTER_ARTICLES=$(jq -nc --argjson kinds '[30023]' --arg since "$(since_days $DAYS_ARTICLES)" ' - {kinds:$kinds, since: ($since|tonumber)}') - - FILTER_REPLIES_E=$(jq -nc --argjson kinds '[1]' --argjson es "$ARTICLE_E_LIST" --arg since "$(since_days $DAYS_THREADS)" ' - {kinds:$kinds, "#e":$es, since: ($since|tonumber)}') - - FILTER_REPLIES_A=$(jq -nc --argjson kinds '[1]' --argjson as "$ARTICLE_A_LIST" --arg since "$(since_days $DAYS_THREADS)" ' - {kinds:$kinds, "#a":$as, since: ($since|tonumber)}') - - FILTER_REACTS=$(jq -nc --argjson kinds '[7]' --argjson es "$ARTICLE_E_LIST" '{kinds:$kinds, "#e":$es}') - FILTER_ZAPS=$(jq -nc --argjson kinds '[9735]' --argjson es "$ARTICLE_E_LIST" '{kinds:$kinds, "#e":$es}') - FILTER_HL=$(jq -nc --argjson kinds '[9802]' --argjson as "$ARTICLE_A_LIST" '{kinds:$kinds, "#a":$as}') - FILTER_PROFILES=$(jq -nc --argjson kinds '[0]' '{kinds:$kinds}') - FILTER_DELETES=$(jq -nc --argjson kinds '[5]' --arg since "$(since_days 30)" '{kinds:$kinds, since:($since|tonumber)}') -else - # Fallback to basic JSON strings - SINCE_ARTICLES=$(since_days $DAYS_ARTICLES) - SINCE_THREADS=$(since_days $DAYS_THREADS) - SINCE_DELETES=$(since_days 30) - - FILTER_ARTICLES="{\"kinds\":[30023],\"since\":${SINCE_ARTICLES}}" - FILTER_REPLIES_E="{\"kinds\":[1],\"#e\":${ARTICLE_E_LIST},\"since\":${SINCE_THREADS}}" - FILTER_REPLIES_A="{\"kinds\":[1],\"#a\":${ARTICLE_A_LIST},\"since\":${SINCE_THREADS}}" - FILTER_REACTS="{\"kinds\":[7],\"#e\":${ARTICLE_E_LIST}}" - FILTER_ZAPS="{\"kinds\":[9735],\"#e\":${ARTICLE_E_LIST}}" - FILTER_HL="{\"kinds\":[9802],\"#a\":${ARTICLE_A_LIST}}" - FILTER_PROFILES="{\"kinds\":[0]}" - FILTER_DELETES="{\"kinds\":[5],\"since\":${SINCE_DELETES}}" -fi - -run_sync() { - local upstream=$1 - local filter=$2 - local label=$3 - echo "[$(date)] Syncing ${label} from ${upstream}..." - - # Write filter to temp file to avoid shell escaping nightmares - local tmpfile="/tmp/strfry-filter-$$.json" - echo "$filter" | docker compose exec strfry sh -c "cat > $tmpfile" - - # Run sync with filter file - docker compose exec strfry sh -c "./strfry sync '$upstream' --filter=\$(cat $tmpfile) && rm $tmpfile" || echo "[$(date)] WARNING: sync failed for ${label} from ${upstream}" -} - -# Sync from all upstream relays -for R in $UPSTREAMS; do - echo "[$(date)] Processing relay: ${R}" - run_sync "$R" "$FILTER_ARTICLES" "articles (30023)" - run_sync "$R" "$FILTER_REPLIES_E" "replies by event-id" - run_sync "$R" "$FILTER_REPLIES_A" "replies by a-tag" - run_sync "$R" "$FILTER_REACTS" "reactions (7)" - run_sync "$R" "$FILTER_ZAPS" "zap receipts (9735)" - run_sync "$R" "$FILTER_HL" "highlights (9802)" - run_sync "$R" "$FILTER_PROFILES" "profiles (0)" - run_sync "$R" "$FILTER_DELETES" "deletes (5)" -done - -echo "[$(date)] Relay ingest complete." - diff --git a/bin/relay/prime.sh b/bin/relay/prime.sh deleted file mode 100644 index 88e4821..0000000 --- a/bin/relay/prime.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "[$(date)] Starting relay prime (one-time backfill)..." - -# Larger time windows for initial backfill -export DAYS_ARTICLES=${DAYS_ARTICLES:-90} -export DAYS_THREADS=${DAYS_THREADS:-30} - -# Use the same ingest logic but with extended time windows -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec "${SCRIPT_DIR}/ingest.sh" - diff --git a/bin/relay/test-smoke.php b/bin/relay/test-smoke.php deleted file mode 100644 index 4fd020c..0000000 --- a/bin/relay/test-smoke.php +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env php -connect(); - echo "✓ Connected successfully\n\n"; - - // Test 2: Query for long-form articles (kind 30023) - echo "Test 2: Querying for kind:30023 events (limit 1)...\n"; - - $filter = new Filter(); - $filter->setKinds([30023]); - $filter->setLimit(1); - - $subscriptionId = 'test-' . bin2hex(random_bytes(8)); - $requestMessage = new RequestMessage($subscriptionId, [$filter]); - - $client = $relay->getClient(); - $client->setTimeout(10); - $client->text($requestMessage->generate()); - - $foundEvent = false; - $eventCount = 0; - $startTime = time(); - $timeout = 10; - - while ((time() - $startTime) < $timeout) { - try { - $response = $client->receive(); - - if (!$response instanceof Text) { - continue; - } - - $content = $response->getContent(); - $decoded = json_decode($content, true); - - if (!is_array($decoded) || count($decoded) < 2) { - continue; - } - - $messageType = $decoded[0] ?? ''; - - if ($messageType === 'EVENT') { - $eventCount++; - $event = $decoded[2] ?? []; - $eventId = $event['id'] ?? 'unknown'; - $eventKind = $event['kind'] ?? 'unknown'; - - echo "✓ Received EVENT: id={$eventId}, kind={$eventKind}\n"; - $foundEvent = true; - - // Send CLOSE - $client->text(json_encode(['CLOSE', $subscriptionId])); - break; - } elseif ($messageType === 'EOSE') { - echo " Received EOSE (End of Stored Events)\n"; - // Send CLOSE - $client->text(json_encode(['CLOSE', $subscriptionId])); - break; - } elseif ($messageType === 'NOTICE' || $messageType === 'CLOSED') { - echo " Received {$messageType}: " . ($decoded[1] ?? '') . "\n"; - break; - } - } catch (TimeoutException $e) { - echo " Timeout waiting for response\n"; - break; - } - } - - if (!$foundEvent && $eventCount === 0) { - echo "⚠ No events found (relay might be empty - try running 'make relay-prime' first)\n\n"; - } else { - echo "\n"; - } - - // Test 3: Verify write rejection - echo "Test 3: Testing write policy (should reject)...\n"; - // We'll just document this - actual test would require creating a signed event - echo "⚠ Write rejection test not implemented (requires event signing)\n"; - echo " Manual test: Try publishing an event - should receive rejection message\n\n"; - - $relay->disconnect(); - - echo str_repeat('-', 60) . "\n"; - echo "✓ Smoke test completed successfully\n"; - - exit(0); - -} catch (\Exception $e) { - echo "\n✗ ERROR: " . $e->getMessage() . "\n"; - echo "Stack trace:\n" . $e->getTraceAsString() . "\n"; - exit(1); -} - diff --git a/bin/relay/verify-config.php b/bin/relay/verify-config.php deleted file mode 100644 index 2f8b532..0000000 --- a/bin/relay/verify-config.php +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env php - strfry relay ### strfry: image: dockurr/strfry:latest - container_name: newsroom-strfry restart: unless-stopped - environment: - - RELAY_NAME=Decent Newsroom Read-Cache Relay - - RELAY_DESCRIPTION=Read-only cache relay for Decent Newsroom - - RELAY_WRITE_POLICY=/app/write-policy.sh + command: + - /bin/sh + - -c + - | + ./strfry relay /etc/strfry.conf & + ./strfry router /etc/router.conf & + wait volumes: - - ./infra/strfry/write-policy.sh:/app/write-policy.sh:ro - - strfry_data:/app/strfry-db + - ./docker/strfry/strfry.conf:/etc/strfry.conf:ro + - ./docker/strfry/write-policy.sh:/app/write-policy.sh:ro + - ./docker/strfry/router.conf:/etc/router.conf:ro + - strfry_data:/var/lib/strfry ports: - - "7777:7777" # Expose for local testing (ws://localhost:7777) - healthcheck: - test: ["CMD-SHELL", "timeout 3 bash -c '> /var/log/cron.log 2>&1 -0 */2 * * * /media_discovery.sh >> /var/log/cron.log 2>&1 +2 */2 * * * /media_discovery.sh >> /var/log/cron.log 2>&1 0 */2 * * * /article_discovery.sh >> /var/log/cron.log 2>&1 diff --git a/docker/strfry/router.conf b/docker/strfry/router.conf new file mode 100644 index 0000000..09ea81d --- /dev/null +++ b/docker/strfry/router.conf @@ -0,0 +1,35 @@ +# Same DB path the relay uses +db = "/var/lib/strfry" + +streams { + # One named stream group that pulls down exactly the kinds you care about + ingest { + dir = "down" + + # Pull long-form + replies + reactions + zaps + highlights + hygiene + # 30023 = NIP-23 article + # 30024 = NIP-23 draft + # 1 = replies/comments + # 1111 = comments + # 7 = reactions + # 9735 = zap receipts + # 9802 = highlights + # 0 = profiles + # 5 = deletes + filter = {"kinds":[30023,30024,1111,9735,9802,0,5]} + + urls = [ + "wss://nos.lol" + "wss://relay.damus.io" + "wss://theforest.nostr1.com" + ] + } + + # If you later want a second policy (e.g., only profiles), add another block: + # { "kinds": [30023, 1111, 9802] } + # profiles_only { + # dir = "down" + # filter = { "kinds": [0] } + # urls = [ "wss://nos.lol" ] + # } +} diff --git a/infra/strfry/strfry.conf b/docker/strfry/strfry.conf similarity index 92% rename from infra/strfry/strfry.conf rename to docker/strfry/strfry.conf index 3a9b6fd..9e96456 100644 --- a/infra/strfry/strfry.conf +++ b/docker/strfry/strfry.conf @@ -3,7 +3,7 @@ ## # Directory that contains the strfry LMDB database (restart required) -db = "/app/strfry-db" +db = "/var/lib/strfry" dbParams { # Maximum number of threads/processes that can simultaneously have LMDB transactions open (restart required) @@ -54,13 +54,13 @@ relay { name = "Decent Newsroom Read-Cache Relay" # NIP-11: Detailed information about relay, free-form - description = "Read-only cache relay; denies client writes. Crawls upstream relays for long-form articles and related activity." + description = "Read-only cache relay for long-form articles and related activity." # NIP-11: Administrative nostr pubkey, for contact purposes - pubkey = "" + pubkey = "d475ce4b3977507130f42c7f86346ef936800f3ae74d5ecf8089280cdc1923e9" # NIP-11: Alternative administrative contact (email, website, etc) - contact = "" + contact = "decentnewsroom.com" } # Maximum accepted incoming websocket frame size (should be larger than max event) (restart required) @@ -83,7 +83,7 @@ relay { writePolicy { # If non-empty, path to an executable script that implements the writePolicy plugin logic - plugin = "/app/write-policy.sh" + # plugin = "/opt/write-policy.sh" } compression { diff --git a/infra/strfry/write-policy.sh b/docker/strfry/write-policy.sh similarity index 100% rename from infra/strfry/write-policy.sh rename to docker/strfry/write-policy.sh diff --git a/infra/cron/Dockerfile b/infra/cron/Dockerfile deleted file mode 100644 index cfecbc5..0000000 --- a/infra/cron/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM alpine:3.19 - -# Install supercronic, bash, jq, docker-cli for the ingest script -RUN apk add --no-cache \ - bash \ - curl \ - jq \ - docker-cli \ - && curl -fsSLO https://github.com/aptible/supercronic/releases/download/v0.2.29/supercronic-linux-amd64 \ - && chmod +x supercronic-linux-amd64 \ - && mv supercronic-linux-amd64 /usr/local/bin/supercronic - -# Set working directory -WORKDIR /app - - -# Default command (will be overridden by compose) -CMD ["/usr/local/bin/supercronic", "/etc/cron/crontab"] - diff --git a/infra/cron/crontab b/infra/cron/crontab deleted file mode 100644 index 99acca1..0000000 --- a/infra/cron/crontab +++ /dev/null @@ -1,6 +0,0 @@ -# Relay ingest crontab - syncs events from upstream relays every 10 minutes -# Format: minute hour day month weekday command - -# Run ingest every 10 minutes -*/10 * * * * /app/bin/relay/ingest.sh >> /var/log/relay-ingest.log 2>&1 - diff --git a/infra/strfry/Dockerfile b/infra/strfry/Dockerfile deleted file mode 100644 index 59b681c..0000000 --- a/infra/strfry/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM debian:bookworm-slim - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - git \ - build-essential \ - libtool \ - autotools-dev \ - automake \ - pkg-config \ - liblmdb-dev \ - libsecp256k1-dev \ - libzstd-dev \ - libssl-dev \ - wget \ - && rm -rf /var/lib/apt/lists/* - -# Build strfry from source -WORKDIR /tmp -RUN git clone --depth 1 --branch v1.0.6 https://github.com/hoytech/strfry.git && \ - cd strfry && \ - git submodule update --init && \ - make setup-golpe && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf /tmp/strfry - -# Create data directory -RUN mkdir -p /var/strfry/db - -# Expose relay port -EXPOSE 7777 - -# Default command (can be overridden) -CMD ["strfry", "relay"] - diff --git a/sync-strfry.sh b/sync-strfry.sh new file mode 100644 index 0000000..800e375 --- /dev/null +++ b/sync-strfry.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker compose exec strfry ./strfry sync wss://theforest.nostr1.com --filter '{"kinds":[9802,1111,30023,0]}' --dir down +docker compose exec strfry ./strfry sync wss://relay.damus.io --filter '{"kinds":[9802,1111,30023,0]}' --dir down