diff --git a/PROXY_SETUP.md b/PROXY_SETUP.md index 4bb062a2..28e187e4 100644 --- a/PROXY_SETUP.md +++ b/PROXY_SETUP.md @@ -185,7 +185,7 @@ VITE_LANGUAGE_TOOL_URL=/api/languagetool VITE_TRANSLATE_URL=/api/translate ``` -**Production:** `docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compose.prod.yml up -d` starts the full stack (including LanguageTool on **127.0.0.1:8010** and LibreTranslate on **127.0.0.1:5000**). Run `bash scripts/ensure-libretranslate-dirs.sh` once on the server for LibreTranslate volume permissions. Proxy `/api/languagetool` and `/api/translate` from Apache/nginx to those ports, and bake the client with `LANGUAGE_TOOL_URL=/api/languagetool` and `TRANSLATE_URL=/api/translate` when running `./scripts/build-and-push-prod.sh`. +**Production:** `docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compose.prod.yml up -d` starts the full stack (including LanguageTool on **127.0.0.1:8010** and LibreTranslate on **127.0.0.1:5000**). Run `bash scripts/ensure-libretranslate-dirs.sh` once on the server (LibreTranslate UID **1032** on `.local-libretranslate`, Piper ONNX into `.local-piper-data` and the **`_piper-stack-data`** Docker volume when it exists). Proxy `/api/languagetool` and `/api/translate` from Apache/nginx to those ports, and bake the client with `LANGUAGE_TOOL_URL=/api/languagetool` and `TRANSLATE_URL=/api/translate` when running `./scripts/build-and-push-prod.sh`. **Notes:** LanguageTool’s JVM image often needs **~1–2 GiB** RAM. LibreTranslate **does not listen on port 5000 until models are ready**; without **`LT_LOAD_ONLY`** it may pull **many gigabytes** first, so the Vite proxy can show **`ECONNRESET` on `/translate`** while booting. Compose defaults **`LT_LOAD_ONLY`** to **ten** widely used codes (**en, de, es, fr, it, pt, ru, zh, ja, ar** — see `libretranslate` in `docker-compose.dev.yml`). Override with **`LT_LOAD_ONLY`** to add or remove codes; first start downloads packs for every listed code. **`LT_UPDATE_MODELS`** defaults to **`true`** so if you **expand** `LT_LOAD_ONLY` later, a **recreated** container still **installs missing** Argos packages into the bind-mounted `.local-libretranslate` tree (otherwise an older en/de-only cache sticks). Set **`LT_UPDATE_MODELS=false`** after everything is installed if you want faster routine restarts. Models are stored under **`.local-libretranslate/share`** and **`.local-libretranslate/cache`** (gitignored) with **bind mounts** so they survive **`docker compose down`**, image updates, and container recreate. **`scripts/ensure-libretranslate-dirs.sh`** (run automatically by **`npm run dev:all`**, **`npm run stack:remote`**, **`npm run docker:editor-tools`**, etc.) creates those dirs and **`chown`s them to UID 1032** via a short **Alpine** container so the LibreTranslate user can write. If you start **`libretranslate` by hand**, run **`npm run docker:prep-libretranslate`** once first. First download can still take **several minutes**; use **`docker logs -f jumble-libretranslate`** until **`curl http://127.0.0.1:5000/languages`** returns JSON. If logs show **`Cannot update models`** / **`Unavailable language codes: …`**, one bad token in **`LT_LOAD_ONLY` aborts the whole install** (you stay on whatever was already on disk, often en/de only). **Norwegian** must be **`nb`** (Bokmål), not ISO **`no`**. After you shrink **`LT_LOAD_ONLY`**, run **`npm run docker:prune-libretranslate-packages`** to remove leftover Argos package dirs under **`.local-libretranslate/share/argos-translate/packages`** (and unused **MiniSBD** `.onnx` files); the script briefly stops **`jumble-libretranslate`** or **`imwald-libretranslate`**. Override codes with **`LT_LOAD_ONLY=…`** on the same command if they differ from the compose default. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8df1dd1e..e51311d5 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -72,8 +72,7 @@ services: # Install missing packs when `LT_LOAD_ONLY` grows (persisted volume otherwise keeps old en/de-only index). LT_UPDATE_MODELS: ${LT_UPDATE_MODELS:-true} volumes: - - ./.local-libretranslate/share:/home/libretranslate/.local/share - - ./.local-libretranslate/cache:/home/libretranslate/.local/cache + - ./.local-libretranslate:/home/libretranslate/.local networks: - jumble restart: unless-stopped diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 8876c99a..a3066480 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,7 +1,7 @@ # Production stack: jumble + NIP-66 monitor + og-proxy + Wyoming Piper + Piper HTTP proxy + LanguageTool + LibreTranslate. # `docker compose up -d` starts every service here (including piper-tts-proxy) if images exist; it only pulls images that are missing locally. To refresh :latest first: `docker compose pull` or `docker compose up -d --pull always`. # Remote: docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compose.prod.yml up -d -# First-time LibreTranslate bind mounts: bash scripts/ensure-libretranslate-dirs.sh (see scripts/README-deploy.md). +# First-time LibreTranslate + Piper models: bash scripts/ensure-libretranslate-dirs.sh (see scripts/README-deploy.md). # Images built/pushed by ./scripts/build-and-push-prod.sh (app, monitor, piper-tts-proxy); others pull from Hub. # # Apache (unchanged on your host) should keep (order: specific paths before catch-all /): @@ -87,8 +87,8 @@ services: LT_LOAD_ONLY: ${LT_LOAD_ONLY:-en,de,es,fr,it,pt,ru,zh,ja,ar} LT_UPDATE_MODELS: ${LT_UPDATE_MODELS:-true} volumes: - - ./.local-libretranslate/share:/home/libretranslate/.local/share - - ./.local-libretranslate/cache:/home/libretranslate/.local/cache + # One tree under .local (same as upstream run.sh -v …/lt-local:/home/libretranslate/.local); avoids split-mount permission edge cases. + - ./.local-libretranslate:/home/libretranslate/.local restart: unless-stopped deploy: resources: diff --git a/docker-compose.yml b/docker-compose.yml index f77a7956..ada38407 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,8 +53,7 @@ services: LT_LOAD_ONLY: ${LT_LOAD_ONLY:-en,de,es,fr,it,pt,ru,zh,ja,ar} LT_UPDATE_MODELS: ${LT_UPDATE_MODELS:-true} volumes: - - ./.local-libretranslate/share:/home/libretranslate/.local/share - - ./.local-libretranslate/cache:/home/libretranslate/.local/cache + - ./.local-libretranslate:/home/libretranslate/.local networks: - jumble restart: unless-stopped diff --git a/package-lock.json b/package-lock.json index 4f1343b5..216f4276 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "imwald", - "version": "23.0.2", + "version": "23.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "imwald", - "version": "23.0.2", + "version": "23.0.4", "license": "MIT", "dependencies": { "@asciidoctor/core": "^3.0.4", diff --git a/package.json b/package.json index cf705e55..fa02991a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "imwald", - "version": "23.0.3", + "version": "23.0.4", "description": "Imwald — a user-friendly Nostr client focused on relay feed browsing, publications, and relay discovery", "private": true, "type": "module", diff --git a/scripts/README-deploy.md b/scripts/README-deploy.md index cd9398f0..1622a58a 100644 --- a/scripts/README-deploy.md +++ b/scripts/README-deploy.md @@ -37,10 +37,11 @@ Registry paths keep the historical `imwald-jumble` name; retagging is optional a NIP66_MONITOR_NPUB=npub1... ``` -4. **Once per machine** (LibreTranslate bind mounts need UID 1032 on the host dirs): +4. **Once per machine** (LibreTranslate UID 1032 + Piper ONNX into `.local-piper-data` and, if present, the `_piper-stack-data` Docker volume): ```bash bash scripts/ensure-libretranslate-dirs.sh ``` + Large download; re-runs skip existing `.onnx` pairs. Use `SKIP_PIPER_VOICES=1` to only fix LibreTranslate dirs. If the compose project name is not the repo folder name, set `COMPOSE_PROJECT_NAME` so the Piper volume matches (e.g. `jumble_piper-stack-data`). ## Remote server: pull and run diff --git a/scripts/build-and-push-prod.sh b/scripts/build-and-push-prod.sh index 46fa9ea0..4f5cbcb1 100755 --- a/scripts/build-and-push-prod.sh +++ b/scripts/build-and-push-prod.sh @@ -76,5 +76,5 @@ git tag -a "$GIT_TAG" -m "Release $GIT_TAG" echo "Pushing tag $GIT_TAG to origin" git push origin "$GIT_TAG" -echo "Done. On the server (repo clone): bash scripts/ensure-libretranslate-dirs.sh # once, for LibreTranslate volume permissions" +echo "Done. On the server (repo clone): bash scripts/ensure-libretranslate-dirs.sh # LibreTranslate perms + Piper ONNX into .local-piper-data and piper-stack-data volume" echo " docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compose.prod.yml up -d" diff --git a/scripts/download-piper-extra-voices.sh b/scripts/download-piper-extra-voices.sh index 52e9db40..8866b66c 100644 --- a/scripts/download-piper-extra-voices.sh +++ b/scripts/download-piper-extra-voices.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash -# Download Piper ONNX voices from rhasspy/piper-voices into .local-piper-data (mounted as /data for piper-wyoming). -# Usage: bash scripts/download-piper-extra-voices.sh +# Download Piper ONNX voices from rhasspy/piper-voices (trinity read-aloud + extras). +# Usage: bash scripts/download-piper-extra-voices.sh [DEST_DIR] +# DEST_DIR defaults to repo/.local-piper-data (Wyoming --data-dir in docker-compose.dev.yml). +# Env: HF_BASE — override Hugging Face resolve base (default rhasspy/piper-voices/main). set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" -DEST="${ROOT}/.local-piper-data" -HF="https://huggingface.co/rhasspy/piper-voices/resolve/main" +DEST="${1:-${PIPER_DOWNLOAD_DIR:-$ROOT/.local-piper-data}}" +HF="${HF_BASE:-https://huggingface.co/rhasspy/piper-voices/resolve/main}" mkdir -p "$DEST" @@ -12,16 +14,34 @@ fetch_pair() { local relpath="$1" local base_name base_name="$(basename "$relpath")" + local onnx="${DEST}/${base_name}.onnx" + local json="${DEST}/${base_name}.onnx.json" + if [[ -f "$onnx" && -f "$json" ]]; then + echo "Skip (exists): ${base_name}" + return 0 + fi echo "Fetching ${base_name} ..." - curl -fsSL -o "${DEST}/${base_name}.onnx" "${HF}/${relpath}.onnx" - curl -fsSL -o "${DEST}/${base_name}.onnx.json" "${HF}/${relpath}.onnx.json" + curl -fsSL -o "$onnx" "${HF}/${relpath}.onnx" + curl -fsSL -o "$json" "${HF}/${relpath}.onnx.json" } -# Paths match rhasspy/piper-voices `voices.json` and EXTRA_READ_ALOUD_PIPER_VOICE / server getVoiceForLanguage. +# --- Trinity UI locales (keep in sync with src/lib/trinity-languages.ts TRINITY_PIPER_VOICE) --- +fetch_pair "en/en_US/lessac/medium/en_US-lessac-medium" +fetch_pair "de/de_DE/thorsten/medium/de_DE-thorsten-medium" +fetch_pair "fr/fr_FR/siwis/medium/fr_FR-siwis-medium" +fetch_pair "es/es_ES/davefx/medium/es_ES-davefx-medium" +fetch_pair "ru/ru_RU/ruslan/medium/ru_RU-ruslan-medium" +fetch_pair "zh/zh_CN/huayan/medium/zh_CN-huayan-medium" +fetch_pair "pl/pl_PL/darkman/medium/pl_PL-darkman-medium" +fetch_pair "nl/nl_NL/mls/medium/nl_NL-mls-medium" +fetch_pair "cs/cs_CZ/jirka/medium/cs_CZ-jirka-medium" +fetch_pair "tr/tr_TR/dfki/medium/tr_TR-dfki-medium" + +# --- Read-aloud extras (EXTRA_READ_ALOUD_PIPER_VOICE + server voiceMap) --- fetch_pair "ar/ar_JO/kareem/medium/ar_JO-kareem-medium" fetch_pair "it/it_IT/paola/medium/it_IT-paola-medium" fetch_pair "pt/pt_BR/cadu/medium/pt_BR-cadu-medium" -# Japanese: rhasspy/piper-voices has no ja_* ONNX; the app still uses Chinese Piper as a CJK-related read-aloud voice for `ja`. +# Japanese: no ja_* in rhasspy/piper-voices; app uses Chinese Piper for ja-related read-aloud. -echo "Done. Files are in ${DEST}. Restart piper-wyoming if it is already running." +echo "Done. Piper files in ${DEST}. Restart piper-wyoming if it is already running." diff --git a/scripts/ensure-libretranslate-dirs.sh b/scripts/ensure-libretranslate-dirs.sh index 454de797..11eb3e58 100755 --- a/scripts/ensure-libretranslate-dirs.sh +++ b/scripts/ensure-libretranslate-dirs.sh @@ -1,10 +1,43 @@ #!/usr/bin/env bash -# Host dirs for LibreTranslate bind mounts must be writable by container UID 1032 (see docker-compose*.yml). -# Uses a one-shot Alpine container so you do not need sudo chown on the host. +# One-shot host prep for editor stack sidecars: +# 1) LibreTranslate bind mount — dirs + ownership UID 1032 (official image user). +# 2) Piper ONNX voices — same set as scripts/download-piper-extra-voices.sh into: +# - ./.local-piper-data (dev compose bind mount), and +# - Docker volume _piper-stack-data when it exists (docker-compose.prod.yml Wyoming /data). +# +# Run from repo root: bash scripts/ensure-libretranslate-dirs.sh +# +# Optional env: +# COMPOSE_PROJECT_NAME — Docker Compose project name (default: basename of repo dir), for volume jumble_piper-stack-data. +# SKIP_PIPER_VOICES=1 — only fix LibreTranslate permissions, do not download Piper (~hundreds of MB). set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PROJECT="${COMPOSE_PROJECT_NAME:-$(basename "$ROOT")}" +PIPER_VOL="${PROJECT}_piper-stack-data" + +echo "[ensure] LibreTranslate data dir (UID 1032) …" mkdir -p "$ROOT/.local-libretranslate/share" "$ROOT/.local-libretranslate/cache" docker run --rm \ - -v "$ROOT/.local-libretranslate/share:/s" \ - -v "$ROOT/.local-libretranslate/cache:/c" \ - alpine:3.20 chown -R 1032:1032 /s /c + -v "$ROOT/.local-libretranslate:/d" \ + alpine:3.20 chown -R 1032:1032 /d + +if [[ "${SKIP_PIPER_VOICES:-}" == "1" ]]; then + echo "[ensure] SKIP_PIPER_VOICES=1 — skipping Piper voice download." + exit 0 +fi + +echo "[ensure] Piper voices (bind mount .local-piper-data) …" +bash "$ROOT/scripts/download-piper-extra-voices.sh" "$ROOT/.local-piper-data" + +if docker volume inspect "$PIPER_VOL" &>/dev/null; then + echo "[ensure] Copying Piper voices into Docker volume ${PIPER_VOL} …" + docker run --rm \ + -v "$PIPER_VOL:/data" \ + -v "$ROOT/.local-piper-data:/src:ro" \ + alpine:3.20 \ + sh -c 'set -e; for f in /src/*.onnx /src/*.onnx.json; do [ -f "$f" ] || continue; bn=$(basename "$f"); cp -a "$f" "/data/$bn"; done; ls -la /data | head -20' +else + echo "[ensure] No Docker volume ${PIPER_VOL} (prod Wyoming uses it). Skipping volume copy — dev-only .local-piper-data is ready." +fi + +echo "[ensure] Done." diff --git a/scripts/stack-remote.sh b/scripts/stack-remote.sh index 0e4f4ae7..1876b064 100644 --- a/scripts/stack-remote.sh +++ b/scripts/stack-remote.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # One remote command from repo clone: full docker-compose.prod.yml stack (pull + up). -# Same as: ensure-libretranslate-dirs.sh && docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compose.prod.yml up -d +# Same as: ensure-libretranslate-dirs.sh (LT perms + Piper) && docker compose … pull && up -d # First-party images must be pushed from ./scripts/build-and-push-prod.sh before pull will get new app/monitor/piper-proxy revisions. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)"