You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
754 lines
31 KiB
754 lines
31 KiB
#!/bin/bash |
|
|
|
# Import JSONL files to remote Orly relay |
|
# This script copies files from local exports directory to remote server and imports them |
|
# Usage: ./scripts/import-exports-remote.sh [remote_host] [remote_user] |
|
|
|
set -e |
|
|
|
# Configuration |
|
REMOTE_HOST="${REMOTE_HOST:-${1:-217.154.126.125}}" |
|
REMOTE_USER="${REMOTE_USER:-${2:-root}}" |
|
REMOTE_TEMP_DIR="${REMOTE_TEMP_DIR:-/root/tmp/orly}" |
|
DOCKER_CONTAINER="${DOCKER_CONTAINER:-orly-relay}" |
|
# NOSTR_PRIVATE_KEY for localhost authentication (reads from environment, e.g., .bashrc) |
|
NOSTR_PRIVATE_KEY="${NOSTR_PRIVATE_KEY:-}" |
|
|
|
# Colors |
|
GREEN='\033[0;32m' |
|
YELLOW='\033[1;33m' |
|
RED='\033[0;31m' |
|
BLUE='\033[0;34m' |
|
CYAN='\033[0;36m' |
|
NC='\033[0m' |
|
|
|
# Get script directory and calculate exports path |
|
# Script is in scripts/, exports are in ../scripts/exports relative to script |
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
|
EXPORTS_DIR="$(cd "$SCRIPT_DIR/../../scripts/exports" 2>/dev/null && pwd || echo "")" |
|
|
|
# IMPORTANT: This script NEVER deletes source files from EXPORTS_DIR |
|
# It only copies files to remote server and moves them to a "done" folder after successful import |
|
|
|
# Helper function to query an event ID from the relay |
|
# Returns 0 if event found, 1 if not found |
|
query_event_on_relay() { |
|
local event_id="$1" |
|
local relay_ws_http="ws://${REMOTE_HOST}:7777/" |
|
local relay_ws_https="wss://${REMOTE_HOST}:7777/" |
|
local event_found=0 |
|
|
|
for relay_ws in "$relay_ws_http" "$relay_ws_https"; do |
|
if command -v websocat >/dev/null 2>&1; then |
|
local query_msg="[\"REQ\",\"verify-query-$(date +%s)\",{\"ids\":[\"${event_id}\"]}]" |
|
local query_result=$(timeout 10 bash -c "echo '$query_msg' | websocat '$relay_ws' 2>&1" | head -20) |
|
|
|
if echo "$query_result" | grep -q "\"id\":\"${event_id}\""; then |
|
return 0 |
|
fi |
|
elif command -v python3 >/dev/null 2>&1; then |
|
python3 <<PYEOF |
|
import json |
|
import sys |
|
import asyncio |
|
import ssl |
|
try: |
|
import websockets |
|
except ImportError: |
|
sys.exit(1) |
|
|
|
async def query_event(relay_url, event_id): |
|
try: |
|
ssl_context = None |
|
if relay_url.startswith('wss://'): |
|
ssl_context = ssl.create_default_context() |
|
ssl_context.check_hostname = False |
|
ssl_context.verify_mode = ssl.CERT_NONE |
|
|
|
async with websockets.connect(relay_url, timeout=10, ssl=ssl_context) as ws: |
|
# Read initial messages (AUTH, etc.) but don't return True for them |
|
try: |
|
initial_response = await asyncio.wait_for(ws.recv(), timeout=3.0) |
|
# Just consume it, don't check it |
|
except asyncio.TimeoutError: |
|
pass |
|
except Exception: |
|
pass |
|
|
|
# Send REQ for the specific event ID |
|
req_msg = ["REQ", f"verify-query-{int(__import__('time').time())}", {"ids": [event_id]}] |
|
await ws.send(json.dumps(req_msg)) |
|
|
|
# Wait for responses - we need to find the actual EVENT with matching ID |
|
# Keep reading until we get EOSE or timeout |
|
found_event = False |
|
try: |
|
while True: |
|
response = await asyncio.wait_for(ws.recv(), timeout=5.0) |
|
data = json.loads(response) |
|
if isinstance(data, list) and len(data) > 0: |
|
if data[0] == "EVENT" and len(data) > 2: |
|
event_data = data[2] |
|
if isinstance(event_data, dict) and event_data.get("id") == event_id: |
|
found_event = True |
|
break |
|
elif data[0] == "EOSE": |
|
# End of stored events - stop waiting |
|
break |
|
except asyncio.TimeoutError: |
|
pass |
|
except Exception: |
|
pass |
|
|
|
return found_event |
|
except Exception: |
|
return False |
|
|
|
relay_url = "${relay_ws}" |
|
event_id = "${event_id}" |
|
result = asyncio.run(query_event(relay_url, event_id)) |
|
if result: |
|
sys.exit(0) |
|
else: |
|
sys.exit(1) |
|
PYEOF |
|
if [ $? -eq 0 ]; then |
|
return 0 |
|
fi |
|
fi |
|
done |
|
|
|
return 1 |
|
} |
|
|
|
# Check if exports directory exists |
|
if [ -z "$EXPORTS_DIR" ] || [ ! -d "$EXPORTS_DIR" ]; then |
|
echo -e "${RED}Error: Exports directory not found at: $SCRIPT_DIR/../scripts/exports${NC}" |
|
echo "Expected path: $(cd "$SCRIPT_DIR/.." && pwd)/scripts/exports" |
|
exit 1 |
|
fi |
|
|
|
# Find all JSONL files |
|
jsonl_files=("$EXPORTS_DIR"/*.jsonl) |
|
|
|
# Check if any JSONL files exist |
|
if [ ! -e "${jsonl_files[0]}" ]; then |
|
echo -e "${YELLOW}No JSONL files found in: $EXPORTS_DIR${NC}" |
|
exit 0 |
|
fi |
|
|
|
# Count files |
|
file_count=0 |
|
for file in "${jsonl_files[@]}"; do |
|
if [ -f "$file" ]; then |
|
file_count=$((file_count + 1)) |
|
fi |
|
done |
|
|
|
if [ "$file_count" -eq 0 ]; then |
|
echo -e "${YELLOW}No JSONL files found in: $EXPORTS_DIR${NC}" |
|
exit 0 |
|
fi |
|
|
|
echo -e "${BLUE}=== Import Exports to Remote Server ===${NC}" |
|
echo "" |
|
echo "Exports directory: $EXPORTS_DIR" |
|
echo "Remote server: ${REMOTE_USER}@${REMOTE_HOST}" |
|
echo "Remote temp directory: ${REMOTE_TEMP_DIR}" |
|
echo "Docker container: ${DOCKER_CONTAINER}" |
|
echo "Files found: $file_count" |
|
echo "" |
|
|
|
# Check if SSH is available |
|
if ! command -v ssh >/dev/null 2>&1; then |
|
echo -e "${RED}Error: ssh command not found. Please install OpenSSH client.${NC}" |
|
exit 1 |
|
fi |
|
|
|
# Check if SCP is available |
|
if ! command -v scp >/dev/null 2>&1; then |
|
echo -e "${RED}Error: scp command not found. Please install OpenSSH client.${NC}" |
|
exit 1 |
|
fi |
|
|
|
# Test SSH connection |
|
echo -e "${CYAN}Testing SSH connection to ${REMOTE_USER}@${REMOTE_HOST}...${NC}" |
|
if ! ssh -o ConnectTimeout=10 -o BatchMode=yes "${REMOTE_USER}@${REMOTE_HOST}" "echo 'Connection successful'" 2>/dev/null; then |
|
echo -e "${YELLOW}Warning: SSH connection test failed. You may be prompted for password/key.${NC}" |
|
echo "Continuing anyway..." |
|
fi |
|
echo "" |
|
|
|
# Check if Docker container exists on remote |
|
echo -e "${CYAN}Checking Docker container on remote server...${NC}" |
|
if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --format '{{.Names}}' | grep -q '^${DOCKER_CONTAINER}$'" 2>/dev/null; then |
|
echo -e "${RED}Error: Docker container '${DOCKER_CONTAINER}' not found on remote server.${NC}" |
|
echo "Available containers:" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker ps --format '{{.Names}}'" 2>/dev/null || true |
|
exit 1 |
|
fi |
|
echo -e "${GREEN}✓ Docker container '${DOCKER_CONTAINER}' found${NC}" |
|
echo "" |
|
|
|
# Check if NOSTR_PRIVATE_KEY is available (local or in container) |
|
echo -e "${CYAN}Checking for NOSTR_PRIVATE_KEY...${NC}" |
|
container_has_key=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} sh -c 'echo \$NOSTR_PRIVATE_KEY'" 2>/dev/null | grep -v '^$' || echo "") |
|
|
|
if [ -n "$container_has_key" ]; then |
|
echo -e "${GREEN}✓ NOSTR_PRIVATE_KEY: Found in container (will use for localhost authentication)${NC}" |
|
elif [ -n "$NOSTR_PRIVATE_KEY" ]; then |
|
echo -e "${GREEN}✓ NOSTR_PRIVATE_KEY: Found in local environment${NC}" |
|
echo -e "${CYAN} Will pass it to container for localhost authentication${NC}" |
|
else |
|
echo -e "${YELLOW}⚠ NOSTR_PRIVATE_KEY: Not found${NC}" |
|
echo -e "${YELLOW} Set it locally: export NOSTR_PRIVATE_KEY=nsec1your_key_here${NC}" |
|
echo -e "${YELLOW} Or add to container environment when running docker${NC}" |
|
fi |
|
echo "" |
|
|
|
# Create temp directory on remote (needed for pre-flight test) |
|
echo -e "${CYAN}Creating temporary directory on remote server...${NC}" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p '${REMOTE_TEMP_DIR}'" || { |
|
echo -e "${RED}Error: Failed to create temp directory on remote server${NC}" |
|
exit 1 |
|
} |
|
echo -e "${GREEN}✓ Temp directory created${NC}" |
|
echo "" |
|
|
|
# Pre-flight test: Send a test event and verify it appears on the relay |
|
# This runs BEFORE copying files to catch configuration issues early |
|
echo -e "${CYAN}=== Pre-flight Test: Verifying Import Works ===${NC}" |
|
echo " Creating and sending a test event..." |
|
echo " (This verifies configuration before copying large files)" |
|
|
|
# Determine which NOSTR_PRIVATE_KEY to use |
|
TEST_NOSTR_KEY="" |
|
if [ -n "$container_has_key" ]; then |
|
TEST_NOSTR_KEY="$container_has_key" |
|
echo " Using NOSTR_PRIVATE_KEY from container for test event" |
|
elif [ -n "$NOSTR_PRIVATE_KEY" ]; then |
|
TEST_NOSTR_KEY="$NOSTR_PRIVATE_KEY" |
|
echo " Using NOSTR_PRIVATE_KEY from local environment for test event" |
|
else |
|
echo -e "${RED}✗ NOSTR_PRIVATE_KEY not found - cannot create signed test event${NC}" |
|
echo -e "${RED}✗ Pre-flight test FAILED - Need NOSTR_PRIVATE_KEY for write permissions${NC}" |
|
exit 1 |
|
fi |
|
|
|
# Create a test event using Go |
|
TEST_EVENT_FILE=$(mktemp /tmp/orly-test-event-XXXXXX.jsonl) |
|
TEST_EVENT_ID="" |
|
|
|
# Get the script directory to find the Go program |
|
GO_PROGRAM="${SCRIPT_DIR}/create-test-event.go" |
|
|
|
# Compile and run the Go program |
|
if command -v go >/dev/null 2>&1; then |
|
echo " Creating signed test event with Go..." |
|
|
|
# Compile the Go program to a temporary binary |
|
TEMP_BINARY=$(mktemp /tmp/orly-test-event-XXXXXX) |
|
TEMP_ERR=$(mktemp /tmp/orly-test-event-err-XXXXXX) |
|
|
|
if ! go build -o "$TEMP_BINARY" "$GO_PROGRAM" 2>"$TEMP_ERR"; then |
|
COMPILE_ERR=$(cat "$TEMP_ERR" 2>/dev/null || echo "") |
|
echo -e "${RED}✗ Failed to compile test event generator${NC}" |
|
if [ -n "$COMPILE_ERR" ]; then |
|
echo " Compile error: $COMPILE_ERR" |
|
fi |
|
rm -f "$TEST_EVENT_FILE" "$TEMP_BINARY" "$TEMP_ERR" |
|
exit 1 |
|
fi |
|
rm -f "$TEMP_ERR" |
|
|
|
# Run it with NOSTR_PRIVATE_KEY |
|
TEST_EVENT_JSON=$(NOSTR_PRIVATE_KEY="$TEST_NOSTR_KEY" "$TEMP_BINARY" 2>"$TEMP_ERR") |
|
EXIT_CODE=$? |
|
ERROR_OUTPUT=$(cat "$TEMP_ERR" 2>/dev/null || echo "") |
|
|
|
# Clean up binary and error file |
|
rm -f "$TEMP_BINARY" "$TEMP_ERR" |
|
|
|
if [ $EXIT_CODE -ne 0 ] || [ -z "$TEST_EVENT_JSON" ]; then |
|
echo -e "${RED}✗ Failed to create signed test event${NC}" |
|
if [ -n "$ERROR_OUTPUT" ]; then |
|
echo " Error: $ERROR_OUTPUT" |
|
fi |
|
rm -f "$TEST_EVENT_FILE" |
|
exit 1 |
|
fi |
|
|
|
# Save the event JSON |
|
echo "$TEST_EVENT_JSON" > "$TEST_EVENT_FILE" |
|
|
|
# Extract event ID |
|
if command -v jq >/dev/null 2>&1; then |
|
TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | jq -r '.id' 2>/dev/null || echo "") |
|
elif command -v python3 >/dev/null 2>&1; then |
|
TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | python3 -c "import sys, json; print(json.load(sys.stdin).get('id', ''))" 2>/dev/null || echo "") |
|
else |
|
# Fallback: extract ID using grep/sed |
|
TEST_EVENT_ID=$(echo "$TEST_EVENT_JSON" | grep -o '"id":"[^"]*"' | sed 's/"id":"\([^"]*\)"/\1/' | head -1) |
|
fi |
|
else |
|
echo -e "${RED}✗ Go compiler not found - cannot create signed test event${NC}" |
|
echo -e "${YELLOW} Install Go: https://golang.org/dl/${NC}" |
|
exit 1 |
|
fi |
|
|
|
if [ -z "$TEST_EVENT_ID" ] || [ ! -s "$TEST_EVENT_FILE" ]; then |
|
echo -e "${RED}✗ Failed to create test event or extract event ID${NC}" |
|
rm -f "$TEST_EVENT_FILE" |
|
exit 1 |
|
fi |
|
|
|
echo " Test event ID: ${TEST_EVENT_ID:0:16}..." |
|
echo " Uploading test event to container..." |
|
|
|
# Copy test event to remote server first, then into container |
|
REMOTE_TEMP_EVENT="/tmp/orly-test-event-$(date +%s).jsonl" |
|
if ! scp "$TEST_EVENT_FILE" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_TEMP_EVENT}" 2>&1; then |
|
echo -e "${RED}✗ Failed to copy test event to remote server${NC}" |
|
rm -f "$TEST_EVENT_FILE" |
|
exit 1 |
|
fi |
|
|
|
# Now copy from remote server into container |
|
if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker cp ${REMOTE_TEMP_EVENT} ${DOCKER_CONTAINER}:/tmp/test_event.jsonl" 2>&1; then |
|
echo -e "${RED}✗ Failed to copy test event to container${NC}" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_TEMP_EVENT}" 2>/dev/null || true |
|
rm -f "$TEST_EVENT_FILE" |
|
exit 1 |
|
fi |
|
|
|
# Fix ownership and permissions so the orly user can read it |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u root ${DOCKER_CONTAINER} chown orly:orly /tmp/test_event.jsonl && docker exec -u root ${DOCKER_CONTAINER} chmod 644 /tmp/test_event.jsonl" 2>/dev/null || true |
|
|
|
# Clean up remote temp file |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_TEMP_EVENT}" 2>/dev/null || true |
|
|
|
# Send test event via import API |
|
echo " Sending test event via import API..." |
|
REMOTE_RESPONSE_FILE="/tmp/orly-test-response-$(date +%s).txt" |
|
|
|
if [ -n "$container_has_key" ]; then |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time 25 --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/test_event.jsonl\" \"http://127.0.0.1:7777/api/import?async=true\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1 |
|
CURL_EXIT=$? |
|
elif [ -n "$NOSTR_PRIVATE_KEY" ]; then |
|
escaped_key=$(echo "$NOSTR_PRIVATE_KEY" | sed "s/'/'\\\\''/g") |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} sh -c \"export NOSTR_PRIVATE_KEY=\\\"$escaped_key\\\" && curl --max-time 25 --connect-timeout 5 -s -w \\\"\\\\n%{http_code}\\\" -X POST -F \\\"file=@/tmp/test_event.jsonl\\\" \\\"http://127.0.0.1:7777/api/import?async=true\\\"\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1 |
|
CURL_EXIT=$? |
|
else |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout 30 bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time 25 --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/test_event.jsonl\" \"http://127.0.0.1:7777/api/import?async=true\" > ${REMOTE_RESPONSE_FILE} 2>&1'" 2>&1 |
|
CURL_EXIT=$? |
|
fi |
|
|
|
# Retrieve the response file |
|
test_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_RESPONSE_FILE} 2>/dev/null" 2>/dev/null || echo "") |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_RESPONSE_FILE}" 2>/dev/null || true |
|
|
|
# Check if curl timed out or failed |
|
if [ $CURL_EXIT -eq 124 ]; then |
|
echo -e "${RED}✗ Test upload timed out after 30 seconds${NC}" |
|
rm -f "$TEST_EVENT_FILE" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true |
|
exit 1 |
|
elif [ $CURL_EXIT -ne 0 ]; then |
|
echo -e "${RED}✗ Test upload failed (exit code: $CURL_EXIT)${NC}" |
|
rm -f "$TEST_EVENT_FILE" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true |
|
exit 1 |
|
fi |
|
|
|
test_http_code=$(echo "$test_response" | tail -n1 | tr -d '[:space:]') |
|
test_body=$(echo "$test_response" | sed '$d') |
|
|
|
if [ "$test_http_code" -ne 200 ] && [ "$test_http_code" -ne 202 ]; then |
|
echo -e "${RED}✗ Test upload failed (HTTP $test_http_code)${NC}" |
|
rm -f "$TEST_EVENT_FILE" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true |
|
exit 1 |
|
fi |
|
|
|
echo -e "${GREEN}✓ Test event uploaded successfully (HTTP $test_http_code)${NC}" |
|
echo " Waiting 5 seconds for event to be processed..." |
|
sleep 5 |
|
|
|
echo " Querying relay for test event..." |
|
RELAY_WS_HTTP="ws://${REMOTE_HOST}:7777/" |
|
RELAY_WS_HTTPS="wss://${REMOTE_HOST}:7777/" |
|
RELAY_WS="" |
|
EVENT_FOUND=0 |
|
|
|
# Try to query using websocat or Python |
|
for RELAY_WS in "$RELAY_WS_HTTP" "$RELAY_WS_HTTPS"; do |
|
echo " Trying ${RELAY_WS}..." |
|
|
|
if command -v websocat >/dev/null 2>&1; then |
|
QUERY_MSG="[\"REQ\",\"test-query-$(date +%s)\",{\"ids\":[\"${TEST_EVENT_ID}\"]}]" |
|
QUERY_RESULT=$(timeout 10 bash -c "echo '$QUERY_MSG' | websocat '$RELAY_WS' 2>&1" | head -20) |
|
|
|
if echo "$QUERY_RESULT" | grep -q "\"id\":\"${TEST_EVENT_ID}\""; then |
|
EVENT_FOUND=1 |
|
break |
|
fi |
|
elif command -v python3 >/dev/null 2>&1; then |
|
echo " Using Python websockets to query..." |
|
QUERY_RESULT=$(python3 <<PYEOF |
|
import json |
|
import sys |
|
import asyncio |
|
import ssl |
|
try: |
|
import websockets |
|
except ImportError: |
|
print("ERROR: websockets library not installed", file=sys.stderr) |
|
sys.exit(1) |
|
|
|
async def query_event(relay_url, event_id): |
|
try: |
|
ssl_context = None |
|
if relay_url.startswith('wss://'): |
|
ssl_context = ssl.create_default_context() |
|
ssl_context.check_hostname = False |
|
ssl_context.verify_mode = ssl.CERT_NONE |
|
|
|
async with websockets.connect(relay_url, timeout=10, ssl=ssl_context) as ws: |
|
# Read initial messages (AUTH, etc.) but don't return True for them |
|
try: |
|
initial_response = await asyncio.wait_for(ws.recv(), timeout=3.0) |
|
# Just consume it, don't check it |
|
except asyncio.TimeoutError: |
|
pass |
|
except Exception: |
|
pass |
|
|
|
# Send REQ for the specific event ID |
|
req_msg = ["REQ", f"test-query-{int(__import__('time').time())}", {"ids": [event_id]}] |
|
await ws.send(json.dumps(req_msg)) |
|
|
|
# Wait for responses - we need to find the actual EVENT with matching ID |
|
# Keep reading until we get EOSE or timeout |
|
found_event = False |
|
try: |
|
while True: |
|
response = await asyncio.wait_for(ws.recv(), timeout=5.0) |
|
data = json.loads(response) |
|
if isinstance(data, list) and len(data) > 0: |
|
if data[0] == "EVENT" and len(data) > 2: |
|
event_data = data[2] |
|
if isinstance(event_data, dict) and event_data.get("id") == event_id: |
|
found_event = True |
|
break |
|
elif data[0] == "EOSE": |
|
# End of stored events - stop waiting |
|
break |
|
except asyncio.TimeoutError: |
|
pass |
|
except Exception: |
|
pass |
|
|
|
return found_event |
|
except Exception: |
|
return False |
|
|
|
relay_url = "${RELAY_WS}" |
|
event_id = "${TEST_EVENT_ID}" |
|
result = asyncio.run(query_event(relay_url, event_id)) |
|
if result: |
|
sys.exit(0) |
|
else: |
|
sys.exit(1) |
|
PYEOF |
|
) |
|
QUERY_EXIT=$? |
|
|
|
if [ $QUERY_EXIT -eq 0 ]; then |
|
EVENT_FOUND=1 |
|
break |
|
fi |
|
else |
|
echo -e "${YELLOW} Warning: Neither websocat nor Python websockets available${NC}" |
|
break |
|
fi |
|
done |
|
|
|
# Clean up test files |
|
rm -f "$TEST_EVENT_FILE" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f /tmp/test_event.jsonl" 2>/dev/null || true |
|
|
|
if [ "$EVENT_FOUND" -eq 1 ]; then |
|
echo -e "${GREEN}✓ Test event found on relay${NC}" |
|
echo -e "${GREEN}✓ Pre-flight test PASSED - Import is working${NC}" |
|
else |
|
echo -e "${RED}✗ Test event NOT found on relay${NC}" |
|
echo -e "${RED}✗ Pre-flight test FAILED - Import is not working${NC}" |
|
echo -e "${YELLOW} Fix configuration issues before proceeding${NC}" |
|
exit 1 |
|
fi |
|
|
|
echo "" |
|
echo -e "${GREEN}✓ All pre-flight checks passed. Proceeding with file copy and import...${NC}" |
|
echo "" |
|
|
|
# Process files one at a time: copy, extract event ID, import, verify, move to done |
|
echo -e "${BLUE}=== Processing Files One at a Time ===${NC}" |
|
echo -e "${CYAN}Method: Using HTTP API import (synchronous, no container restart needed)${NC}" |
|
echo "" |
|
|
|
success_count=0 |
|
fail_count=0 |
|
current=0 |
|
DONE_DIR="${REMOTE_TEMP_DIR}/done" |
|
|
|
# Ensure temp and done directories exist on remote |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p '${REMOTE_TEMP_DIR}' '${DONE_DIR}'" 2>/dev/null || true |
|
|
|
# Process each local file one at a time |
|
for local_file in "${jsonl_files[@]}"; do |
|
if [ ! -f "$local_file" ]; then |
|
continue |
|
fi |
|
|
|
current=$((current + 1)) |
|
filename=$(basename "$local_file") |
|
file_size=$(du -h "$local_file" | cut -f1) |
|
remote_file="${REMOTE_TEMP_DIR}/${filename}" |
|
|
|
echo "" |
|
echo -e "${BLUE}=== [$current/$file_count] Processing: $filename (${file_size}) ===${NC}" |
|
echo "" |
|
|
|
# Step 1: Copy file to remote server |
|
echo -e "${CYAN}[1/5] Copying file to remote server...${NC}" |
|
if ! scp "$local_file" "${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_TEMP_DIR}/" 2>&1; then |
|
echo -e "${RED}✗ Failed to copy: $filename${NC}" |
|
fail_count=$((fail_count + 1)) |
|
echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" |
|
continue |
|
fi |
|
echo -e "${GREEN}✓ Copied: $filename${NC}" |
|
echo "" |
|
|
|
# Step 2: Extract event ID from the remote file |
|
echo -e "${CYAN}[2/5] Extracting event ID from file...${NC}" |
|
file_event_id="" |
|
|
|
# Extract first event ID from the file (look for "id":"..." pattern) |
|
# Try multiple patterns to handle different JSON formats |
|
file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -oE '\"id\"[[:space:]]*:[[:space:]]*\"[a-f0-9]{64}\"' | head -1 | sed -E 's/\"id\"[[:space:]]*:[[:space:]]*\"([^\"]+)\"/\1/'" || echo "") |
|
|
|
# If that didn't work, try simpler pattern |
|
if [ -z "$file_event_id" ]; then |
|
file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -o '\"id\":\"[a-f0-9]\{64\}\"' | head -1 | sed 's/\"id\":\"\([^\"]*\)\"/\1/'" || echo "") |
|
fi |
|
|
|
# If still no ID, try to find any 64-char hex string that looks like an event ID |
|
if [ -z "$file_event_id" ]; then |
|
file_event_id=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "head -1 '${remote_file}' 2>/dev/null | grep -oE '[a-f0-9]{64}' | head -1" || echo "") |
|
fi |
|
|
|
if [ -n "$file_event_id" ]; then |
|
echo -e "${GREEN}✓ Extracted event ID: ${file_event_id:0:16}...${NC}" |
|
else |
|
echo -e "${YELLOW}⚠ Could not extract event ID (will skip verification)${NC}" |
|
fi |
|
echo "" |
|
|
|
# Step 3: Import the file via HTTP API |
|
echo -e "${CYAN}[3/5] Importing file via HTTP API...${NC}" |
|
|
|
# Check file size first |
|
file_size_bytes=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "stat -c%s '${remote_file}' 2>/dev/null" || echo "0") |
|
file_size_mb=$(echo "scale=2; $file_size_bytes / 1024 / 1024" | bc 2>/dev/null || echo "0") |
|
echo " File size: ${file_size_mb} MB" |
|
|
|
import_success=false |
|
|
|
# Copy file into container for HTTP API import |
|
echo " Copying file into container..." |
|
if ! ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker cp '${remote_file}' ${DOCKER_CONTAINER}:/tmp/" 2>&1; then |
|
echo -e "${RED}✗ Failed to copy file into container${NC}" |
|
fail_count=$((fail_count + 1)) |
|
echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" |
|
continue |
|
fi |
|
|
|
# Fix ownership and permissions |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u root ${DOCKER_CONTAINER} chown orly:orly /tmp/${filename} && docker exec -u root ${DOCKER_CONTAINER} chmod 644 /tmp/${filename}" 2>/dev/null || true |
|
|
|
# Calculate timeout based on file size |
|
timeout_seconds=$((file_size_bytes / 1024 / 1024 * 30)) # ~30 seconds per MB |
|
if [ $timeout_seconds -lt 300 ]; then |
|
timeout_seconds=300 # Minimum 5 minutes |
|
elif [ $timeout_seconds -gt 7200 ]; then |
|
timeout_seconds=7200 # Maximum 2 hours |
|
fi |
|
echo " Using timeout: ${timeout_seconds}s for ${file_size_mb} MB file" |
|
echo "" |
|
|
|
# Start streaming container logs in background to show import progress |
|
# Use prominent log prefix so it's easy to find in logs |
|
LOG_PREFIX="[IMPORT ${filename}]" |
|
echo -e "${CYAN} Starting import via HTTP API (synchronous mode)...${NC}" |
|
echo -e "${CYAN} Streaming container logs with prefix: ${LOG_PREFIX}${NC}" |
|
echo "" |
|
|
|
# Start log streaming in background |
|
LOG_TAIL_PID="" |
|
(ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker logs -f --tail 0 ${DOCKER_CONTAINER} 2>&1" | while IFS= read -r line; do |
|
# Show all lines with [HTTP API IMPORT] prefix (our new prominent logging) |
|
# Also show other import-related lines for context |
|
if echo "$line" | grep -qiE "\[HTTP API IMPORT\]|import.*progress|import.*complete|import.*failed|import.*error"; then |
|
echo -e "${CYAN} ${LOG_PREFIX} $line${NC}" >&2 |
|
fi |
|
done) & |
|
LOG_TAIL_PID=$! |
|
sleep 1 # Give log stream a moment to start |
|
|
|
# Get NOSTR_PRIVATE_KEY for authentication |
|
container_has_key=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} sh -c 'echo \$NOSTR_PRIVATE_KEY'" 2>/dev/null | grep -v '^$' || echo "") |
|
|
|
REMOTE_IMPORT_RESPONSE="/tmp/orly-import-response-$(date +%s).txt" |
|
|
|
# Show start time |
|
UPLOAD_START=$(date +%s) |
|
echo " [$(date +%H:%M:%S)] Starting HTTP API import..." |
|
echo -e "${CYAN} ${LOG_PREFIX} Import started at $(date)${NC}" |
|
|
|
# Run import via HTTP API (synchronous - waits for completion) |
|
if [ -n "$container_has_key" ]; then |
|
echo " Using NOSTR_PRIVATE_KEY from container (localhost authentication)" |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/${filename}\" \"http://127.0.0.1:7777/api/import\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1 |
|
CURL_EXIT=$? |
|
elif [ -n "$NOSTR_PRIVATE_KEY" ]; then |
|
echo " Passing NOSTR_PRIVATE_KEY to container for localhost authentication" |
|
escaped_key=$(echo "$NOSTR_PRIVATE_KEY" | sed "s/'/'\\\\''/g") |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} sh -c \"export NOSTR_PRIVATE_KEY=\\\"$escaped_key\\\" && curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \\\"\\\\n%{http_code}\\\" -X POST -F \\\"file=@/tmp/${filename}\\\" \\\"http://127.0.0.1:7777/api/import\\\"\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1 |
|
CURL_EXIT=$? |
|
else |
|
echo -e "${YELLOW} Warning: No NOSTR_PRIVATE_KEY available${NC}" |
|
echo " Trying without authentication (may fail if ACL requires auth)..." |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "timeout ${timeout_seconds} bash -c 'docker exec -u orly ${DOCKER_CONTAINER} curl --max-time $((timeout_seconds - 5)) --connect-timeout 5 -s -w \"\\n%{http_code}\" -X POST -F \"file=@/tmp/${filename}\" \"http://127.0.0.1:7777/api/import\" > ${REMOTE_IMPORT_RESPONSE} 2>&1'" 2>&1 |
|
CURL_EXIT=$? |
|
fi |
|
|
|
UPLOAD_END=$(date +%s) |
|
UPLOAD_DURATION=$((UPLOAD_END - UPLOAD_START)) |
|
echo " [$(date +%H:%M:%S)] HTTP request completed in ${UPLOAD_DURATION}s" |
|
|
|
# Stop log streaming |
|
if [ -n "$LOG_TAIL_PID" ] && kill -0 "$LOG_TAIL_PID" 2>/dev/null; then |
|
kill "$LOG_TAIL_PID" 2>/dev/null || true |
|
wait "$LOG_TAIL_PID" 2>/dev/null || true |
|
fi |
|
|
|
# Get response |
|
import_response=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "cat ${REMOTE_IMPORT_RESPONSE} 2>/dev/null" 2>/dev/null || echo "") |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "rm -f ${REMOTE_IMPORT_RESPONSE}" 2>/dev/null || true |
|
|
|
# Clean up file from container |
|
ssh "${REMOTE_USER}@${REMOTE_HOST}" "docker exec -u orly ${DOCKER_CONTAINER} rm -f '/tmp/${filename}'" 2>/dev/null || true |
|
|
|
# Check if curl timed out |
|
if [ $CURL_EXIT -eq 124 ]; then |
|
echo -e "${RED}✗ Import timed out after ${timeout_seconds} seconds${NC}" |
|
echo -e "${CYAN} ${LOG_PREFIX} Import timed out${NC}" |
|
fail_count=$((fail_count + 1)) |
|
echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" |
|
continue |
|
elif [ $CURL_EXIT -ne 0 ]; then |
|
echo -e "${RED}✗ Import failed (exit code: $CURL_EXIT)${NC}" |
|
echo -e "${CYAN} ${LOG_PREFIX} Import failed with exit code $CURL_EXIT${NC}" |
|
fail_count=$((fail_count + 1)) |
|
echo -e "${YELLOW} Skipping this file and continuing to next...${NC}" |
|
continue |
|
fi |
|
|
|
# Extract HTTP status code and body |
|
import_http_code=$(echo "$import_response" | tail -n1 | tr -d '[:space:]') |
|
import_body=$(echo "$import_response" | sed '$d') |
|
|
|
# Check if import was successful |
|
if [ -z "$import_http_code" ] || ! [[ "$import_http_code" =~ ^[0-9]+$ ]]; then |
|
fail_count=$((fail_count + 1)) |
|
echo -e "${RED}✗ Failed to import: $filename (invalid HTTP response)${NC}" |
|
echo -e "${CYAN} ${LOG_PREFIX} Invalid HTTP response${NC}" |
|
elif [ "$import_http_code" -eq 200 ]; then |
|
if echo "$import_body" | grep -qi '"success":\s*true\|"message".*completed'; then |
|
echo -e "${GREEN}✓ Imported: $filename (HTTP $import_http_code)${NC}" |
|
echo -e "${CYAN} ${LOG_PREFIX} Import completed successfully${NC}" |
|
import_success=true |
|
success_count=$((success_count + 1)) |
|
else |
|
fail_count=$((fail_count + 1)) |
|
echo -e "${RED}✗ Failed to import: $filename (HTTP $import_http_code, but no success in response)${NC}" |
|
echo -e "${CYAN} ${LOG_PREFIX} Import response indicates failure${NC}" |
|
fi |
|
else |
|
fail_count=$((fail_count + 1)) |
|
echo -e "${RED}✗ Failed to import: $filename (HTTP $import_http_code)${NC}" |
|
echo -e "${CYAN} ${LOG_PREFIX} Import failed with HTTP $import_http_code${NC}" |
|
if [ -n "$import_body" ]; then |
|
echo " Response: $import_body" |
|
fi |
|
fi |
|
|
|
# Step 4: Verify import |
|
if [ "$import_success" = true ]; then |
|
echo "" |
|
echo -e "${CYAN}[4/5] Verifying import...${NC}" |
|
if [ -n "$file_event_id" ]; then |
|
echo " Verifying imported event exists on relay..." |
|
sleep 3 # Give relay a moment to process the imported events |
|
if query_event_on_relay "$file_event_id"; then |
|
echo -e "${GREEN}✓ Verified: Event ${file_event_id:0:16}... from ${filename} found on relay${NC}" |
|
else |
|
echo -e "${YELLOW}⚠ Warning: Event ${file_event_id:0:16}... from ${filename} not yet found on relay (may need more time)${NC}" |
|
fi |
|
else |
|
echo -e "${YELLOW}⚠ Skipping verification (no event ID extracted)${NC}" |
|
fi |
|
else |
|
echo "" |
|
echo -e "${CYAN}[4/5] Skipping verification (import failed)${NC}" |
|
fi |
|
echo "" |
|
|
|
# Step 5: Move to done folder (only if import was successful) |
|
if [ "$import_success" = true ]; then |
|
echo -e "${CYAN}[5/5] Moving file to done folder...${NC}" |
|
if ssh "${REMOTE_USER}@${REMOTE_HOST}" "test -f '${remote_file}'" 2>/dev/null; then |
|
if ssh "${REMOTE_USER}@${REMOTE_HOST}" "mv '${remote_file}' '${DONE_DIR}/${filename}'" 2>/dev/null; then |
|
echo -e "${GREEN}✓ Moved: ${filename} to ${DONE_DIR}${NC}" |
|
else |
|
echo -e "${YELLOW}⚠ Warning: Could not move ${filename} to done folder${NC}" |
|
fi |
|
else |
|
echo -e "${YELLOW}⚠ Warning: File ${filename} not found on remote (may have been moved already)${NC}" |
|
fi |
|
else |
|
echo -e "${CYAN}[5/5] Skipping move to done (import failed - file remains in ${REMOTE_TEMP_DIR})${NC}" |
|
fi |
|
echo "" |
|
done |
|
|
|
# Summary |
|
echo "" |
|
echo -e "${BLUE}=== Import Summary ===${NC}" |
|
echo -e "${GREEN}Files succeeded: $success_count${NC}" |
|
if [ "$fail_count" -gt 0 ]; then |
|
echo -e "${RED}Files failed: $fail_count${NC}" |
|
fi |
|
echo "" |
|
|
|
if [ "$fail_count" -eq 0 ]; then |
|
echo -e "${GREEN}All files imported successfully!${NC}" |
|
echo -e "${GREEN}All successful files have been moved to ${DONE_DIR}${NC}" |
|
exit 0 |
|
else |
|
echo -e "${YELLOW}Some imports failed. Check the errors above.${NC}" |
|
echo -e "${YELLOW}Failed files remain in ${REMOTE_TEMP_DIR} for retry${NC}" |
|
echo -e "${GREEN}Successful files have been moved to ${DONE_DIR}${NC}" |
|
exit 1 |
|
fi
|
|
|