next.orly.dev/pkg/database/tokenize.go

package database

import (
	"strings"
	"unicode"

	sha "github.com/minio/sha256-simd"
)

// TokenHashes extracts unique word hashes (8-byte truncated sha256) from content.
// Rules:
// - Unicode-aware: words are sequences of letters or numbers.
// - Lowercased using unicode case mapping.
// - Ignore URLs (starting with http://, https://, www., or containing "://").
// - Ignore nostr: URIs and #[n] mentions.
// - Ignore words shorter than 2 runes.
// - Exclude 64-character hexadecimal strings (likely IDs/pubkeys).
func TokenHashes(content []byte) [][]byte {
	s := string(content)
	var out [][]byte
	seen := make(map[string]struct{})

	i := 0
	for i < len(s) {
		r, size := rune(s[i]), 1
		if r >= 0x80 {
			r, size = utf8DecodeRuneInString(s[i:])
		}

		// Skip whitespace
		if unicode.IsSpace(r) {
			i += size
			continue
		}

		// Skip URLs and schemes
		if hasPrefixFold(s[i:], "http://") || hasPrefixFold(s[i:], "https://") || hasPrefixFold(s[i:], "nostr:") || hasPrefixFold(s[i:], "www.") {
			i = skipUntilSpace(s, i)
			continue
		}
		// If token contains "://" ahead, treat as URL and skip to space
		if j := strings.Index(s[i:], "://"); j == 0 || (j > 0 && isWordStart(r)) {
			// Only if it's at start of token
			before := s[i : i+j]
			if len(before) == 0 || allAlphaNum(before) {
				i = skipUntilSpace(s, i)
				continue
			}
		}
		// Skip #[n] mentions
		if r == '#' && i+size < len(s) && s[i+size] == '[' {
			end := strings.IndexByte(s[i:], ']')
			if end >= 0 {
				i += end + 1
				continue
			}
		}

		// Collect a word
		start := i
		var runes []rune
		for i < len(s) {
			r2, size2 := rune(s[i]), 1
			if r2 >= 0x80 {
				r2, size2 = utf8DecodeRuneInString(s[i:])
			}
			if unicode.IsLetter(r2) || unicode.IsNumber(r2) {
				runes = append(runes, unicode.ToLower(r2))
				i += size2
				continue
			}
			break
		}
		// If we didn't consume any rune for a word, advance by one rune to avoid stalling
		if i == start {
			_, size2 := utf8DecodeRuneInString(s[i:])
			i += size2
			continue
		}
		if len(runes) >= 2 {
			w := string(runes)
			// Exclude 64-char hex strings
			if isHex64(w) {
				continue
			}
			if _, ok := seen[w]; !ok {
				seen[w] = struct{}{}
				h := sha.Sum256([]byte(w))
				out = append(out, h[:8])
			}
		}
	}
	return out
}

func hasPrefixFold(s, prefix string) bool {
	if len(s) < len(prefix) {
		return false
	}
	for i := 0; i < len(prefix); i++ {
		c := s[i]
		p := prefix[i]
		if c == p {
			continue
		}
		// ASCII case-insensitive
		if 'A' <= c && c <= 'Z' {
			c = c - 'A' + 'a'
		}
		if 'A' <= p && p <= 'Z' {
			p = p - 'A' + 'a'
		}
		if c != p {
			return false
		}
	}
	return true
}

func skipUntilSpace(s string, i int) int {
	for i < len(s) {
		r, size := rune(s[i]), 1
		if r >= 0x80 {
			r, size = utf8DecodeRuneInString(s[i:])
		}
		if unicode.IsSpace(r) {
			return i
		}
		i += size
	}
	return i
}

func allAlphaNum(s string) bool {
	for _, r := range s {
		if !(unicode.IsLetter(r) || unicode.IsNumber(r)) {
			return false
		}
	}
	return true
}

func isWordStart(r rune) bool { return unicode.IsLetter(r) || unicode.IsNumber(r) }

// Minimal utf8 rune decode without importing utf8 to avoid extra deps elsewhere
func utf8DecodeRuneInString(s string) (r rune, size int) {
	// Fallback to standard library if available; however, using basic decoding
	for i := 1; i <= 4 && i <= len(s); i++ {
		r, size = rune(s[0]), 1
		if r < 0x80 {
			return r, 1
		}
		// Use stdlib for correctness
		return []rune(s[:i])[0], len(string([]rune(s[:i])[0]))
	}
	return rune(s[0]), 1
}

// isHex64 returns true if s is exactly 64 hex characters (0-9, a-f)
func isHex64(s string) bool {
	if len(s) != 64 {
		return false
	}
	for i := 0; i < 64; i++ {
		c := s[i]
		if c >= '0' && c <= '9' {
			continue
		}
		if c >= 'a' && c <= 'f' {
			continue
		}
		if c >= 'A' && c <= 'F' {
			continue
		}
		return false
	}
	return true
}