You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
308 lines
8.6 KiB
308 lines
8.6 KiB
#!/bin/bash |
|
# Badger Database Migration Script |
|
# Migrates ORLY database to new Badger configuration with VLogPercentile optimization |
|
|
|
set -e # Exit on error |
|
|
|
# Colors for output |
|
RED='\033[0;31m' |
|
GREEN='\033[0;32m' |
|
YELLOW='\033[1;33m' |
|
NC='\033[0m' # No Color |
|
|
|
echo -e "${GREEN}=== ORLY Badger Database Migration ===${NC}" |
|
echo "" |
|
|
|
# Configuration |
|
DATA_DIR="${ORLY_DATA_DIR:-$HOME/.local/share/ORLY}" |
|
BACKUP_DIR="${DATA_DIR}-backup-$(date +%Y%m%d-%H%M%S)" |
|
EXPORT_FILE="${DATA_DIR}/events-export.jsonl" |
|
RELAY_BIN="${RELAY_BIN:-./orly}" |
|
|
|
# Check if relay binary exists |
|
if [ ! -f "$RELAY_BIN" ]; then |
|
echo -e "${RED}Error: ORLY binary not found at $RELAY_BIN${NC}" |
|
echo "Please build the relay first: go build -o orly" |
|
echo "Or set RELAY_BIN environment variable to the binary location" |
|
exit 1 |
|
fi |
|
|
|
# Check if database exists |
|
if [ ! -d "$DATA_DIR" ]; then |
|
echo -e "${YELLOW}Warning: Database directory not found at $DATA_DIR${NC}" |
|
echo "Nothing to migrate. If this is a fresh install, you can skip migration." |
|
exit 0 |
|
fi |
|
|
|
# Check disk space |
|
DB_SIZE=$(du -sb "$DATA_DIR" | cut -f1) |
|
AVAILABLE_SPACE=$(df "$HOME" | tail -1 | awk '{print $4}') |
|
AVAILABLE_SPACE=$((AVAILABLE_SPACE * 1024)) # Convert to bytes |
|
REQUIRED_SPACE=$((DB_SIZE * 3)) # 3x for safety (export + backup + new DB) |
|
|
|
echo "Database size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)" |
|
echo "Available space: $(numfmt --to=iec-i --suffix=B $AVAILABLE_SPACE)" |
|
echo "Required space: $(numfmt --to=iec-i --suffix=B $REQUIRED_SPACE)" |
|
echo "" |
|
|
|
if [ $AVAILABLE_SPACE -lt $REQUIRED_SPACE ]; then |
|
echo -e "${RED}Error: Not enough disk space!${NC}" |
|
echo "Required: $(numfmt --to=iec-i --suffix=B $REQUIRED_SPACE)" |
|
echo "Available: $(numfmt --to=iec-i --suffix=B $AVAILABLE_SPACE)" |
|
echo "" |
|
echo "Options:" |
|
echo " 1. Free up disk space" |
|
echo " 2. Use natural compaction (no migration needed)" |
|
echo " 3. Export to external drive and import back" |
|
exit 1 |
|
fi |
|
|
|
# Check if relay is running |
|
if pgrep -x "orly" > /dev/null; then |
|
echo -e "${YELLOW}Warning: ORLY relay is currently running${NC}" |
|
echo "The relay should be stopped before migration." |
|
echo "" |
|
read -p "Stop the relay now? (y/N) " -n 1 -r |
|
echo |
|
if [[ $REPLY =~ ^[Yy]$ ]]; then |
|
echo "Attempting to stop relay..." |
|
if systemctl is-active --quiet orly; then |
|
sudo systemctl stop orly |
|
echo -e "${GREEN}Relay stopped via systemd${NC}" |
|
else |
|
pkill orly |
|
sleep 2 |
|
if pgrep -x "orly" > /dev/null; then |
|
echo -e "${RED}Failed to stop relay. Please stop it manually and try again.${NC}" |
|
exit 1 |
|
fi |
|
echo -e "${GREEN}Relay stopped${NC}" |
|
fi |
|
else |
|
echo "Please stop the relay and run this script again." |
|
exit 1 |
|
fi |
|
fi |
|
|
|
echo "" |
|
echo -e "${YELLOW}=== Migration Plan ===${NC}" |
|
echo "1. Export all events to JSONL: $EXPORT_FILE" |
|
echo "2. Backup current database to: $BACKUP_DIR" |
|
echo "3. Create new database with optimized configuration" |
|
echo "4. Import all events (rebuilds indexes)" |
|
echo "5. Verify event counts match" |
|
echo "" |
|
echo "Estimated time: $(( (DB_SIZE / 1024 / 1024 / 100) + 1 )) - $(( (DB_SIZE / 1024 / 1024 / 50) + 1 )) minutes" |
|
echo "" |
|
read -p "Proceed with migration? (y/N) " -n 1 -r |
|
echo |
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then |
|
echo "Migration cancelled." |
|
exit 0 |
|
fi |
|
|
|
# Step 1: Export events |
|
echo "" |
|
echo -e "${GREEN}=== Step 1: Exporting Events ===${NC}" |
|
echo "This may take several minutes for large databases..." |
|
echo "" |
|
|
|
# We'll use a Go program to export since the binary doesn't have a CLI export command |
|
# Create temporary export program |
|
EXPORT_PROG=$(mktemp -d)/export-db.go |
|
cat > "$EXPORT_PROG" << 'EOF' |
|
package main |
|
|
|
import ( |
|
"context" |
|
"fmt" |
|
"os" |
|
"next.orly.dev/pkg/database" |
|
) |
|
|
|
func main() { |
|
if len(os.Args) < 3 { |
|
fmt.Fprintf(os.Stderr, "Usage: %s <data-dir> <output-file>\n", os.Args[0]) |
|
os.Exit(1) |
|
} |
|
|
|
dataDir := os.Args[1] |
|
outFile := os.Args[2] |
|
|
|
ctx := context.Background() |
|
cancel := func() {} |
|
|
|
db, err := database.New(ctx, cancel, dataDir, "error") |
|
if err != nil { |
|
fmt.Fprintf(os.Stderr, "Failed to open database: %v\n", err) |
|
os.Exit(1) |
|
} |
|
defer db.Close() |
|
|
|
f, err := os.Create(outFile) |
|
if err != nil { |
|
fmt.Fprintf(os.Stderr, "Failed to create output file: %v\n", err) |
|
os.Exit(1) |
|
} |
|
defer f.Close() |
|
|
|
fmt.Println("Exporting events...") |
|
db.Export(ctx, f) |
|
fmt.Println("Export complete!") |
|
} |
|
EOF |
|
|
|
# Build and run export program |
|
echo "Building export tool..." |
|
EXPORT_BIN=$(mktemp) |
|
if ! go build -o "$EXPORT_BIN" "$EXPORT_PROG" 2>&1; then |
|
echo -e "${RED}Failed to build export tool${NC}" |
|
rm -f "$EXPORT_PROG" "$EXPORT_BIN" |
|
exit 1 |
|
fi |
|
|
|
echo "Running export..." |
|
if ! "$EXPORT_BIN" "$DATA_DIR" "$EXPORT_FILE"; then |
|
echo -e "${RED}Export failed!${NC}" |
|
rm -f "$EXPORT_PROG" "$EXPORT_BIN" |
|
exit 1 |
|
fi |
|
|
|
rm -f "$EXPORT_PROG" "$EXPORT_BIN" |
|
|
|
# Count exported events |
|
EXPORT_COUNT=$(wc -l < "$EXPORT_FILE") |
|
echo -e "${GREEN}Exported $EXPORT_COUNT events${NC}" |
|
echo "Export size: $(du -h "$EXPORT_FILE" | cut -f1)" |
|
|
|
# Step 2: Backup current database |
|
echo "" |
|
echo -e "${GREEN}=== Step 2: Backing Up Current Database ===${NC}" |
|
echo "Moving $DATA_DIR to $BACKUP_DIR" |
|
mv "$DATA_DIR" "$BACKUP_DIR" |
|
echo -e "${GREEN}Backup complete${NC}" |
|
|
|
# Step 3 & 4: Create new database and import |
|
echo "" |
|
echo -e "${GREEN}=== Step 3 & 4: Creating New Database and Importing ===${NC}" |
|
echo "This will take longer as indexes are rebuilt..." |
|
echo "" |
|
|
|
# Create temporary import program |
|
IMPORT_PROG=$(mktemp -d)/import-db.go |
|
cat > "$IMPORT_PROG" << 'EOF' |
|
package main |
|
|
|
import ( |
|
"context" |
|
"fmt" |
|
"os" |
|
"next.orly.dev/pkg/database" |
|
) |
|
|
|
func main() { |
|
if len(os.Args) < 3 { |
|
fmt.Fprintf(os.Stderr, "Usage: %s <data-dir> <import-file>\n", os.Args[0]) |
|
os.Exit(1) |
|
} |
|
|
|
dataDir := os.Args[1] |
|
importFile := os.Args[2] |
|
|
|
ctx := context.Background() |
|
cancel := func() {} |
|
|
|
// This will create new database with updated configuration from database.go |
|
db, err := database.New(ctx, cancel, dataDir, "info") |
|
if err != nil { |
|
fmt.Fprintf(os.Stderr, "Failed to create database: %v\n", err) |
|
os.Exit(1) |
|
} |
|
defer db.Close() |
|
|
|
f, err := os.Open(importFile) |
|
if err != nil { |
|
fmt.Fprintf(os.Stderr, "Failed to open import file: %v\n", err) |
|
os.Exit(1) |
|
} |
|
defer f.Close() |
|
|
|
fmt.Println("Importing events (this may take a while)...") |
|
db.Import(f) |
|
|
|
// Wait for import to complete |
|
fmt.Println("Import started. Waiting for completion...") |
|
fmt.Println("Check the log output above for progress (logged every 100 events)") |
|
} |
|
EOF |
|
|
|
# Build and run import program |
|
echo "Building import tool..." |
|
IMPORT_BIN=$(mktemp) |
|
if ! go build -o "$IMPORT_BIN" "$IMPORT_PROG" 2>&1; then |
|
echo -e "${RED}Failed to build import tool${NC}" |
|
echo "Rolling back..." |
|
mv "$BACKUP_DIR" "$DATA_DIR" |
|
rm -f "$IMPORT_PROG" "$IMPORT_BIN" |
|
exit 1 |
|
fi |
|
|
|
echo "Running import..." |
|
if ! "$IMPORT_BIN" "$DATA_DIR" "$EXPORT_FILE"; then |
|
echo -e "${RED}Import failed!${NC}" |
|
echo "Rolling back..." |
|
rm -rf "$DATA_DIR" |
|
mv "$BACKUP_DIR" "$DATA_DIR" |
|
rm -f "$IMPORT_PROG" "$IMPORT_BIN" |
|
exit 1 |
|
fi |
|
|
|
rm -f "$IMPORT_PROG" "$IMPORT_BIN" |
|
|
|
# Give import goroutine time to process |
|
echo "Waiting for import to complete..." |
|
sleep 10 |
|
|
|
# Step 5: Verify |
|
echo "" |
|
echo -e "${GREEN}=== Step 5: Verification ===${NC}" |
|
|
|
NEW_DB_SIZE=$(du -sb "$DATA_DIR" | cut -f1) |
|
echo "Old database size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)" |
|
echo "New database size: $(numfmt --to=iec-i --suffix=B $NEW_DB_SIZE)" |
|
echo "" |
|
|
|
if [ $NEW_DB_SIZE -lt $((DB_SIZE / 10)) ]; then |
|
echo -e "${YELLOW}Warning: New database is suspiciously small${NC}" |
|
echo "This may indicate an incomplete import." |
|
echo "Check the logs in $DATA_DIR/migration.log" |
|
echo "" |
|
read -p "Continue anyway? (y/N) " -n 1 -r |
|
echo |
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then |
|
echo "Rolling back..." |
|
rm -rf "$DATA_DIR" |
|
mv "$BACKUP_DIR" "$DATA_DIR" |
|
exit 1 |
|
fi |
|
fi |
|
|
|
echo -e "${GREEN}=== Migration Complete! ===${NC}" |
|
echo "" |
|
echo "Summary:" |
|
echo " - Exported: $EXPORT_COUNT events" |
|
echo " - Old DB size: $(numfmt --to=iec-i --suffix=B $DB_SIZE)" |
|
echo " - New DB size: $(numfmt --to=iec-i --suffix=B $NEW_DB_SIZE)" |
|
echo " - Space saved: $(numfmt --to=iec-i --suffix=B $((DB_SIZE - NEW_DB_SIZE)))" |
|
echo " - Backup location: $BACKUP_DIR" |
|
echo "" |
|
echo "Next steps:" |
|
echo " 1. Start the relay: sudo systemctl start orly (or ./orly)" |
|
echo " 2. Monitor performance for 24-48 hours" |
|
echo " 3. Watch for cache hit ratio >85% in logs" |
|
echo " 4. Verify event count and queries work correctly" |
|
echo " 5. After verification, remove backup: rm -rf $BACKUP_DIR" |
|
echo "" |
|
echo "Rollback (if needed):" |
|
echo " Stop relay, then: rm -rf $DATA_DIR && mv $BACKUP_DIR $DATA_DIR" |
|
echo ""
|
|
|