normalize.go

package gist

import (
	"strings"
	"unicode"
	"unicode/utf8"
)

// Normalize is the shared front-end to featurization: it strips HTML tags,
// lowercases, and collapses runs of whitespace. It is deterministic and part of
// the feature pipeline — any change shifts every vector, so it must bump
// [PipelineVersion]. It deliberately keeps more of the text than a near-duplicate
// normalizer would: for topic categorization the words are the signal.
func Normalize(text string) string {
	text = stripTags(text)
	var b strings.Builder
	b.Grow(len(text))
	inSpace := true
	for _, r := range text {
		if unicode.IsSpace(r) {
			if !inSpace {
				b.WriteByte(' ')
				inSpace = true
			}
			continue
		}
		b.WriteRune(unicode.ToLower(r))
		inSpace = false
	}
	return strings.TrimRight(b.String(), " ")
}

// stripTags removes <...> spans, replacing each with a space so words don't fuse
// across a tag boundary. A '<' only opens a tag when it is followed by a tag-like
// byte ([A-Za-z/!?]); otherwise it is kept literally, so ordinary text such as
// "x < y", "5 < 3", or "<3" survives instead of being swallowed to the next '>'
// (or to end-of-input when there is none). Intentionally simple — enough for HTML
// pages/email, not a conformant parser.
func stripTags(s string) string {
	if !strings.ContainsRune(s, '<') {
		return s
	}
	var b strings.Builder
	b.Grow(len(s))
	depth := 0
	for i := 0; i < len(s); {
		r, size := utf8.DecodeRuneInString(s[i:])
		switch {
		case r == '<' && isTagStart(s[i+size:]):
			depth++
		case r == '>' && depth > 0:
			depth--
			b.WriteByte(' ')
		case depth > 0:
			// inside a tag: drop the rune
		default:
			b.WriteRune(r)
		}
		i += size
	}
	return b.String()
}

// isTagStart reports whether s begins with a byte that plausibly opens an HTML
// tag: a letter, or '/' / '!' / '?' for closing tags, comments, and declarations.
// A '<' not followed by one of these is ordinary text, not markup.
func isTagStart(s string) bool {
	if s == "" {
		return false
	}
	c := s[0]
	return c == '/' || c == '!' || c == '?' ||
		('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
}