next.orly.dev/cmd/orly/db/import.go

//go:build !(js && wasm)

package db

import (
	"context"
	"fmt"
	"os"
	"strings"

	"lol.mleku.dev"
	"lol.mleku.dev/chk"
	"lol.mleku.dev/log"

	"next.orly.dev/pkg/database"
)

func runImport(args []string) {
	var inputFile string
	var showHelp bool

	for i, arg := range args {
		if arg == "--file" || arg == "-f" {
			if i+1 < len(args) {
				inputFile = args[i+1]
			}
		} else if arg == "--help" || arg == "-h" {
			showHelp = true
		} else if arg != "" && !strings.HasPrefix(arg, "-") && inputFile == "" {
			inputFile = arg
		}
	}

	if showHelp {
		printImportHelp()
		return
	}

	if inputFile == "" {
		fmt.Fprintln(os.Stderr, "error: input file required")
		fmt.Fprintln(os.Stderr, "usage: orly db import <file.jsonl>")
		fmt.Fprintln(os.Stderr, "   or: orly db import --file <file.jsonl>")
		os.Exit(1)
	}

	cfg := loadConfig()
	lol.SetLogLevel(cfg.LogLevel)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Create database configuration
	dbCfg := &database.DatabaseConfig{
		DataDir:             cfg.DataDir,
		LogLevel:            cfg.LogLevel,
		BlockCacheMB:        cfg.BlockCacheMB,
		IndexCacheMB:        cfg.IndexCacheMB,
		QueryCacheSizeMB:    cfg.QueryCacheSizeMB,
		QueryCacheMaxAge:    cfg.QueryCacheMaxAge,
		QueryCacheDisabled:  cfg.QueryCacheDisabled,
		SerialCachePubkeys:  cfg.SerialCachePubkeys,
		SerialCacheEventIds: cfg.SerialCacheEventIds,
		ZSTDLevel:           cfg.ZSTDLevel,
	}

	// Initialize database directly
	log.I.F("initializing database at %s for import", cfg.DataDir)
	db, err := database.NewWithConfig(ctx, cancel, dbCfg)
	if chk.E(err) {
		log.E.F("failed to initialize database: %v", err)
		os.Exit(1)
	}
	defer db.Close()

	// Wait for database to be ready
	log.I.F("waiting for database to be ready...")
	<-db.Ready()
	log.I.F("database ready")

	// Open input file
	log.I.F("opening input file: %s", inputFile)
	file, err := os.Open(inputFile)
	if chk.E(err) {
		log.E.F("failed to open input file: %v", err)
		os.Exit(1)
	}
	defer file.Close()

	// Get file size for progress
	fileInfo, err := file.Stat()
	if chk.E(err) {
		log.E.F("failed to stat file: %v", err)
		os.Exit(1)
	}
	fileSizeMB := float64(fileInfo.Size()) / 1024 / 1024
	log.I.F("importing %.2f MB from %s", fileSizeMB, inputFile)

	// Import events
	log.I.F("starting import...")
	db.Import(file)

	log.I.F("import completed successfully")
	fmt.Fprintf(os.Stdout, "✓ Imported events from %s (%.2f MB)\n", inputFile, fileSizeMB)
}

func printImportHelp() {
	fmt.Println(`
Import events from a JSONL file directly into the database.

Usage:
  orly db import <file.jsonl>
  orly db import --file <file.jsonl>

This command imports events from a JSONL (JSON Lines) file directly into
the Badger database, bypassing the HTTP API. This is much faster for large
imports when you have direct access to the database.

The input file should contain one JSON event per line (JSONL format).

Environment variables:
  ORLY_DATA_DIR          - Database data directory (required)
  ORLY_DB_LOG_LEVEL      - Log level (default: info)
  ORLY_DB_BLOCK_CACHE_MB - Block cache size in MB (default: 1024)
  ORLY_DB_INDEX_CACHE_MB - Index cache size in MB (default: 512)
  ORLY_DB_ZSTD_LEVEL     - ZSTD compression level (default: 3)

Examples:
  orly db import events.jsonl
  orly db import --file /path/to/large-export.jsonl
`)
}