diff --git a/.cursor/docs/SYSTEM_WALKTHROUGH.md b/.cursor/docs/SYSTEM_WALKTHROUGH.md new file mode 100644 index 0000000..42d2f47 --- /dev/null +++ b/.cursor/docs/SYSTEM_WALKTHROUGH.md @@ -0,0 +1,178 @@ +# Himotoki System Walkthrough + +Agent- and developer-facing map of the Japanese morphological analyzer. Prefer this over stale paths in `docs/AGENTS.md` (e.g. missing `himotoki/models.py`, `scripts/check_segments.py`). + +Deeper background: `docs/ARCHITECTURE.md`, `docs/OPTIMIZATION_REPORT.md`. + +--- + +## 1. What Himotoki is + +Himotoki (紐解き) is a Python port of [ichiran](https://github.com/tshatrov/ichiran). It segments Japanese text into words with readings, glosses, and conjugation trees using: + +1. Dictionary candidate generation (JMdict surfaces + conjugations) +2. Heuristic scoring (`calc_score`, ichiran-compatible) +3. Viterbi-style DP for the best path (`find_best_path`) +4. Grammar overlays (suffixes, synergies/segfilters, counters, splits) + +Runtime depends on a local SQLite DB (`~/.himotoki/himotoki.db` or `data/himotoki.db`, ~**1.8 GB** after index slim / covering indexes; was ~2.94 GB). Persisted trie: `himotoki.trie` (~25 MB) beside the DB. + +--- + +## 2. Package map (where to change what) + +| Concern | Primary location | Notes | +|---------|------------------|-------| +| Public API | `himotoki/__init__.py` | `analyze()`, `warm_up()`, `shutdown()` | +| CLI | `himotoki/cli.py` | `-r/-k/-j/-f`, `setup` | +| Segmentation DP | `himotoki/segment.py` | sticky positions, substring join, Viterbi | +| Types | `himotoki/types.py` | `WordMatch`, `Segment`, `SegmentList`, `CompoundWord`, `ConjData` | +| Scoring | `himotoki/scoring/` | `calc_score`, caches, archaic/UK/POS | +| Dictionary lookup | `himotoki/lookup/` | `find_word*`, `get_conj_data` | +| Grammar | `himotoki/grammar/` | synergies, suffixes, counters, splits | +| Output / WordInfo | `himotoki/output/` | glosses, conjugation display, JSON/text | +| Characters | `himotoki/characters.py` | kana/kanji, mora, romanize | +| Constants | `himotoki/constants.py` | SEQ IDs, conj types (JMdict-version sensitive) | +| Trie filter | `himotoki/trie.py` | marisa-trie of surface forms | +| DB | `himotoki/db/` | models, connection, pragmas | +| Offline load | `himotoki/loading/` | JMdict XML, conjugations, errata | +| Setup | `himotoki/setup.py` | first-run download + DB build | + +**Compatibility shims:** `himotoki/lookup.py`, `output.py`, `synergies.py`, `suffixes.py`, `counters.py`, `splits.py` re-export from the new packages so existing imports keep working. + +### Task → file cheat sheet + +| Task | Touch | +|------|--------| +| Wrong score / ranking | `scoring/calc_score.py`, constants | +| Missing grammar pattern | `grammar/synergy_rules.py` or segfilters | +| Suffix compound (〜ている) | `grammar/suffix_handlers.py` | +| Counter (三匹) | `grammar/counters.py` | +| Segmentation bug | `segment.py` then scoring | +| Gloss / JSON display | `output/` | +| DB schema / indexes | `db/models.py` + rebuild | +| Synthetic entries | `loading/errata.py` | + +--- + +## 3. End-to-end data flow + +``` +Input text + → analyze() / dict_segment() + → segment_text() + → find_sticky_positions() # forbidden boundaries + → find_substring_words() # trie filter + batched IN queries + → find_word_suffix() / counters # compounds not in DB as wholes + → join_substring_words() + → preload_scoring_caches() + → gen_score() → calc_score() + → cull_segments() + → find_best_path() # DP + synergies / segfilters + → fill_segment_path() # WordInfo + gaps + → populate_meanings() # gloss / POS (display) +``` + +Key entry points: + +- `himotoki.analyze(text)` → `segment_text` + `fill_segment_path` +- `output.dict_segment` / `segment_to_json` for CLI formats +- `warm_up()` builds archaic, suffix, counter, and word-trie caches + +--- + +## 4. Database + +### Location + +1. `HIMOTOKI_DB_PATH` / `HIMOTOKI_DB` +2. `~/.himotoki/himotoki.db` if present +3. `data/himotoki.db` (dev) + +### Tables + +| Table | Role | Segmentation-critical? | +|-------|------|------------------------| +| `entry` | seq, root_p, n_kanji/n_kana, primary_nokanji | Yes (metadata; not XML) | +| `kanji_text` / `kana_text` | surface forms + common/ord/best_* | Yes | +| `sense` / `sense_prop` | POS, misc (uk, arch), … | Yes for scoring tags | +| `gloss` | English definitions | Display only | +| `conjugation` / `conj_prop` / `conj_source_reading` | conj chains | Yes | +| `restricted_reading` | kana↔kanji restrictions | Loaded at build; rarely runtime | + +### Hot queries + +- Batched `SELECT … FROM kana_text/kanji_text WHERE text IN (…)` +- Scoring preload: Entry, UK, POS by seq +- `get_conj_data`: conjugations + source readings + props (batched) +- Output: Sense/Gloss for meanings + +Pragmas: WAL, 64MB cache, 256MB mmap (`db/connection.py`). + +### Build pipeline + +1. Parse `JMdict_e.xml` → entries/readings/senses (`loading/jmdict.py`) +2. Generate conjugations from CSV rules (`loading/conjugations.py`) +3. Apply errata (`loading/errata.py`) +4. `ANALYZE` + `VACUUM` + +--- + +## 5. Scoring & equivalence + +“Query quality” means **identical segmentation winners and scores**, not merely valid Japanese. + +Preserve: + +- Candidate `(text, seq, ord, common)` sets +- `calc_score` bonuses/penalties (length coeffs, archaic, weak/skip conj, splits) +- DP: `GAP_PENALTY`, `SCORE_CUTOFF`, synergies/segfilters +- Suffix and counter decomposition + +Safe optimizations: indexes, batched SQL returning the same rows, caches, trie filter (exact membership), dropping unused `entry` XML storage. + +Unsafe: dropping `common`/POS/conj props, changing score constants, approximate matching. + +--- + +## 6. Caches & performance + +| Cache | Module | Notes | +|-------|--------|-------| +| Archaic seq set | `scoring/caches.py` | ~165ms cold | +| Suffix cache | `grammar/suffixes.py` | ~145ms cold | +| Counter cache | `grammar/counters.py` | ~10ms | +| Word trie (+ disk) | `trie.py` | skip full-table rebuild when file fresh | +| LRU Entry/POS/UK/conj/word | `scoring/caches.py` | | +| Meanings | `output/` | display | + +Hot path already uses raw SQL + trie before DB. Remaining cost is often conjugation metadata and DP over candidates. + +--- + +## 7. Tests & commands + +```bash +source .venv/bin/activate +pip install -e . +pytest tests/ -x --tb=short +python scripts/benchmark.py +himotoki setup --yes # rebuild DB after schema changes +``` + +LLM eval tooling lives under `scripts/llm_eval.py` (see root `AGENTS.md`). + +--- + +## 8. Layers (dependency direction) + +``` +cli / __init__ + → output + → segment + → lookup + scoring + grammar + trie + characters + → db +loading / setup → db (offline only) +``` + +Avoid new cycles; prefer lazy imports where historical cycles existed (`lookup` ↔ `splits`). diff --git a/.cursor/rules/himotoki-system.mdc b/.cursor/rules/himotoki-system.mdc new file mode 100644 index 0000000..66d51d5 --- /dev/null +++ b/.cursor/rules/himotoki-system.mdc @@ -0,0 +1,33 @@ +--- +description: Himotoki architecture map — layers, hot path, where to edit. Use when changing segmentation, scoring, DB, or package layout. +alwaysApply: false +--- + +# Himotoki system (short) + +Full walkthrough: `.cursor/docs/SYSTEM_WALKTHROUGH.md` + +## Layers + +- **API/CLI:** `__init__.py`, `cli.py` +- **Segment (DP):** `segment.py` — sticky → substring/trie → score → Viterbi +- **Types:** `types.py` — WordMatch, Segment, CompoundWord, ConjData +- **Scoring:** `scoring/` — `calc_score`, caches (do not change semantics lightly) +- **Lookup:** `lookup/` — find_word, batched `get_conj_data` +- **Grammar:** `grammar/` — synergies, suffixes, counters, splits +- **Output:** `output/` — WordInfo, glosses, conj display +- **DB/load:** `db/`, `loading/`, `setup.py` + +Old module paths (`lookup.py`, `output.py`, `synergies.py`, …) are shims. + +## Equivalence + +Same candidates + same `calc_score` + same DP winners. Safe: indexes, batch SQL, caches, drop unused XML. Unsafe: score/POS/conj data changes. + +## Hot path + +Trie filter → 2× `text IN (...)` on kana/kanji → preload Entry/UK/POS → score → synergies in `find_best_path` → glosses only in output. + +## Verify + +`pytest tests/ -x --tb=short`; after DB schema changes rebuild with `himotoki setup` / `scripts/init_db.py` then measure size. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f6f8bfc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI + +on: + push: + branches: [main, "cursor/**"] + pull_request: + branches: [main] + +jobs: + test: + name: Pytest + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + + - name: Install package (dev) + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Optional dictionary DB + env: + HIMOTOKI_CI_DB_URL: ${{ secrets.HIMOTOKI_CI_DB_URL }} + run: | + # Full dictionary (~1.8GB) is gitignored. Optionally download from a + # private URL configured as repo secret HIMOTOKI_CI_DB_URL so score + # equivalence and DB-backed tests run in CI. + if [ -n "${HIMOTOKI_CI_DB_URL:-}" ]; then + mkdir -p data + echo "Downloading dictionary DB for CI..." + curl -L --fail --retry 3 -o data/himotoki.db "$HIMOTOKI_CI_DB_URL" + ls -lh data/himotoki.db + else + echo "No HIMOTOKI_CI_DB_URL secret; DB-backed tests will skip." + fi + + - name: Run tests + env: + HIMOTOKI_DB_PATH: ${{ github.workspace }}/data/himotoki.db + run: | + pytest tests/ -q --tb=short diff --git a/.gitignore b/.gitignore index d05d96d..32a607b 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ pytest_debug.log *.db-shm *.db-wal *.pgdump +*.trie data/*.db data/*.sqlite data/JMdict_e.xml diff --git a/himotoki/__init__.py b/himotoki/__init__.py index b4309f4..667ac91 100644 --- a/himotoki/__init__.py +++ b/himotoki/__init__.py @@ -349,6 +349,15 @@ async def shutdown_event(): if _executor is not None: _executor.shutdown(wait=True) _executor = None + + # Drop scoring caches so a later restart cannot reuse stale ORM objects + from himotoki.scoring.caches import clear_scoring_caches + clear_scoring_caches() + try: + from himotoki.output.meanings import clear_meanings_cache + clear_meanings_cache() + except Exception: + pass # Close database connection from himotoki.db.connection import close_connection diff --git a/himotoki/counters.py b/himotoki/counters.py index 7539703..2c3538a 100644 --- a/himotoki/counters.py +++ b/himotoki/counters.py @@ -1,914 +1,2 @@ -""" -Counter word recognition module for himotoki. -Ports ichiran's dict-counters.lisp counter functionality. - -Counter expressions combine a number with a counter suffix to form compound words. -Examples: -- 三匹 (sanbiki) = 三 (san, three) + 匹 (hiki, counter for small animals) -- 五冊 (gosatsu) = 五 (go, five) + 冊 (satsu, counter for books) -- 千人 (sennin) = 千 (sen, thousand) + 人 (nin, counter for people) - -This module dynamically generates counter expressions and provides scoring. -""" - -from dataclasses import dataclass, field -from typing import Optional, List, Dict, Tuple, Any, Set, Union -from functools import lru_cache - -from sqlalchemy import select, and_ -from sqlalchemy.orm import Session - -from himotoki.db.models import ( - Entry, KanjiText, KanaText, SenseProp, -) -from himotoki.characters import ( - get_char_class, mora_length, as_hiragana, count_char_class, -) - - -# ============================================================================ -# Japanese Number Parsing -# ============================================================================ - -# Number kanji to value mapping -KANJI_NUMBERS = { - '零': 0, '〇': 0, - '一': 1, '壱': 1, - '二': 2, '弐': 2, - '三': 3, '参': 3, - '四': 4, - '五': 5, - '六': 6, - '七': 7, - '八': 8, - '九': 9, - '十': 10, - '百': 100, - '千': 1000, - '万': 10000, - '億': 100000000, -} - -# Arabic digit to value -DIGIT_VALUES = { - '0': 0, '１': 0, '０': 0, - '1': 1, '１': 1, - '2': 2, '２': 2, - '3': 3, '３': 3, - '4': 4, '４': 4, - '5': 5, '５': 5, - '6': 6, '６': 6, - '7': 7, '７': 7, - '8': 8, '８': 8, - '9': 9, '９': 9, -} - -# Number kana to digit -DIGIT_TO_KANA = { - 1: 'いち', - 2: 'に', - 3: 'さん', - 4: 'よん', # or し - 5: 'ご', - 6: 'ろく', - 7: 'なな', # or しち - 8: 'はち', - 9: 'きゅう', # or く - 10: 'じゅう', - 100: 'ひゃく', - 1000: 'せん', - 10000: 'まん', -} - -# Power values to kana -POWER_TO_KANA = { - 1: 'じゅう', # 10 - 2: 'ひゃく', # 100 - 3: 'せん', # 1000 - 4: 'まん', # 10000 -} - - -def parse_number(text: str) -> Optional[int]: - """ - Parse a Japanese number string into an integer. - - Supports: - - Arabic numerals (1, 2, 3 or １, ２, ３) - - Kanji numerals (一, 二, 三) - - Mixed (十二, 百三十四) - - Arabic + Kanji multiplier (5億, 10万) - - Returns None if the text is not a valid number. - """ - if not text: - return None - - # Try Arabic numerals first - arabic_str = '' - for char in text: - if char in DIGIT_VALUES: - arabic_str += str(DIGIT_VALUES[char]) - elif char.isdigit(): - arabic_str += char - else: - break - - if arabic_str and len(arabic_str) == len(text): - return int(arabic_str) - - # Try Arabic prefix + Kanji multiplier (e.g., 5億, 10万) - if arabic_str: - arabic_value = int(arabic_str) - suffix = text[len(arabic_str):] - - # Check if suffix is a single multiplier - if len(suffix) == 1 and suffix in KANJI_NUMBERS: - multiplier = KANJI_NUMBERS[suffix] - if multiplier >= 10: # 十, 百, 千, 万, 億 - return arabic_value * multiplier - - # Check if suffix is multiple multipliers (e.g., 10万億 -> 10 * 万 * 億) - # Not implemented for now, but could extend if needed - - # Try kanji numerals - return parse_kanji_number(text) - - -def parse_kanji_number(text: str) -> Optional[int]: - """Parse a kanji number string into an integer.""" - if not text: - return None - - result = 0 - current = 0 - - for char in text: - if char not in KANJI_NUMBERS: - return None - - value = KANJI_NUMBERS[char] - - if value >= 10: - # Multiplier (十, 百, 千, 万, 億) - if current == 0: - current = 1 - - if value >= 10000: - # 万 or higher - adds to result - result += current * value - current = 0 - else: - # 十, 百, 千 - multiplies current - current *= value - result += current - current = 0 - else: - # Digit - if current > 0: - result += current - current = value - - # Add any remaining current - result += current - - return result if result > 0 or text == '零' or text == '〇' else None - - -def number_to_kana(n: int, separator: str = '') -> str: - """Convert a number to its kana reading.""" - if n == 0: - return 'ゼロ' - - parts = [] - - # Handle 100000000s (億) - if n >= 100000000: - oku = n // 100000000 - if oku > 1: - parts.append(number_to_kana(oku)) - else: - parts.append('いち') - parts.append('おく') - n %= 100000000 - - # Handle 10000s (万) - if n >= 10000: - man = n // 10000 - if man > 1: - parts.append(number_to_kana(man)) - parts.append('まん') - n %= 10000 - - # Handle 1000s - if n >= 1000: - sen = n // 1000 - if sen == 3: - parts.append('さん') - elif sen > 1: - parts.append(DIGIT_TO_KANA.get(sen, '')) - parts.append('せん') - n %= 1000 - - # Handle 100s - if n >= 100: - hyaku = n // 100 - if hyaku == 3: - parts.append('さん') - parts.append('びゃく') # 300 = sanbyaku - elif hyaku == 6: - parts.append('ろっ') - parts.append('ぴゃく') # 600 = roppyaku - elif hyaku == 8: - parts.append('はっ') - parts.append('ぴゃく') # 800 = happyaku - else: - if hyaku > 1: - parts.append(DIGIT_TO_KANA.get(hyaku, '')) - parts.append('ひゃく') - n %= 100 - - # Handle 10s - if n >= 10: - juu = n // 10 - if juu > 1: - parts.append(DIGIT_TO_KANA.get(juu, '')) - parts.append('じゅう') - n %= 10 - - # Handle 1s - if n > 0: - parts.append(DIGIT_TO_KANA.get(n, '')) - - return separator.join(parts) - - -# ============================================================================ -# Counter Text Class -# ============================================================================ - -@dataclass -class CounterText: - """ - Represents a counter expression (number + counter). - - Ports ichiran's counter-text class from dict-counters.lisp. - - CounterText objects are designed to be compatible with WordMatch - so they can be scored using the main calc_score function. - """ - text: str # Full text (e.g., "三匹") - kana: str # Kana reading (e.g., "さんびき") - number_text: str # Number part (e.g., "三") - number_value: int # Numeric value (e.g., 3) - counter_text: str # Counter part (e.g., "匹") - counter_kana: str # Counter kana (e.g., "ひき") - source: Optional[Union[KanjiText, KanaText]] = None # Source counter entry - ordinalp: bool = False # Is this ordinal? (〜目) - suffix: Optional[str] = None # Kana suffix (e.g., "め" for 目) - _common: Optional[int] = None # Commonness rating (internal) - digit_opts: Optional[Dict] = None # Digit-specific options for phonetic rules - - @property - def seq(self) -> Optional[int]: - """Get the seq from the source counter.""" - return self.source.seq if self.source else None - - @property - def ord(self) -> int: - """Get the ord from the source counter.""" - return self.source.ord if self.source else 0 - - @property - def common(self) -> Optional[int]: - """Get commonness rating from source or override.""" - if self._common is not None: - return self._common - if self.source: - return self.source.common - return 0 - - @property - def word_type(self) -> str: - """Counter expressions are typically kanji.""" - # Check if text contains kanji - if count_char_class(self.text, 'kanji') > 0: - return 'kanji' - return 'kana' - - @property - def reading(self): - """Return self as the reading for compatibility with calc_score.""" - return self - - @property - def conjugations(self) -> None: - """Counters don't have conjugations.""" - return None - - @property - def is_root(self) -> bool: - """Counters are always root forms.""" - return True - - @property - def is_compound(self) -> bool: - """Counters are not compound words.""" - return False - - @property - def components(self) -> List[str]: - """Return empty list for counters (no components).""" - return [] - - def __repr__(self): - return f"" - - -# ============================================================================ -# Counter Cache -# ============================================================================ - -# Cache for counter readings -_counter_cache: Dict[str, List[Tuple[type, Dict]]] = {} -_counter_cache_initialized = False - -# Special counter handling overrides -# Maps seq to digit_opts dict for phonetic rules -# Format: {digit: [opts]} where opts can be 'g' (geminate), 'r' (rendaku), 'h' (handakuten) -# or a string to replace the number reading -SPECIAL_COUNTER_OPTS: Dict[int, Dict] = { - # 匹/疋 (hiki) - counter for small animals - 1583370: {3: ['r']}, # 3 -> さんびき (rendaku) - # 本 (hon) - counter for long thin objects - 1522150: {3: ['r']}, # 3 -> さんぼん (rendaku) - # 杯/盃 (hai) - counter for cups/glasses - 2019640: {3: ['r']}, # 3 -> さんばい (rendaku) - # 階 (kai) - counter for floors - 1203020: {3: ['r']}, # 3 -> さんがい (rendaku) - # 軒 (ken) - counter for houses - 2078590: {3: ['r']}, # 3 -> さんげん (rendaku) - # 遍 (hen) - counter for times - 2208060: {3: ['r']}, # 3 -> さんべん (rendaku) - # 編/篇 (hen) - counter for literary works - 1511870: {3: ['r']}, # 3 -> さんべん (rendaku) - # 足 (soku) - counter for pairs of footwear - 2412230: {3: ['r']}, # 3 -> さんぞく (rendaku) - # 時 (ji) - counter for hours - 2020680: {4: 'よ', 7: 'しち', 9: 'く'}, - # 時間 (jikan) - counter for hours duration - 1315920: {4: 'よ', 9: 'く'}, - # 年 (nen) - counter for years - 2084840: {4: 'よ', 7: 'しち', 9: 'く'}, - # 円 (en) - counter for yen - 1175570: {4: 'よ'}, - # 分 (fun) - counter for minutes - 1502840: {4: ['h']}, # 4 -> よんぷん (handakuten) - # 舗 (ho) - counter for shops - 1514050: {4: ['h']}, - # 敗 (hai) - counter for losses - 1901390: {4: ['h']}, - # 泊 (haku) - counter for nights stay - 1919550: {4: ['h']}, - # 筆 (hitsu) - counter for brush strokes - 1487770: {4: ['h']}, - # 冊 (satsu) - counter for books - uses standard rules - # 人 (nin) - counter for people - special class - 2149890: {4: 'よ', 7: 'しち'}, - # 月 (gatsu) - counter for months - 1255430: {4: 'し', 7: 'しち', 9: 'く'}, -} - -# Special counter classes that need custom kana generation -# Maps seq to a function that returns the kana for a given number -SPECIAL_COUNTER_KANA: Dict[int, callable] = {} - -# Days counter (日 ka) - kun readings for 1-10, 14, 20, 24, 30 -DAYS_KUN_READINGS = { - 1: 'ついたち', - 2: 'ふつか', - 3: 'みっか', - 4: 'よっか', - 5: 'いつか', - 6: 'むいか', - 7: 'なのか', - 8: 'ようか', - 9: 'ここのか', - 10: 'とうか', - 14: 'じゅうよっか', - 20: 'はつか', - 24: 'にじゅうよっか', - 30: 'みそか', -} - -# People counter (人 nin) - kun readings for 1-2 -PEOPLE_KUN_READINGS = { - 1: 'ひとり', - 2: 'ふたり', -} - -SPECIAL_COUNTERS: Dict[int, callable] = {} - -# Extra counter IDs (not marked as ctr but act as counters) -# Note: 時 (2020680) is intentionally not included because it conflicts with -# regular dictionary entries like 三時 (3 o'clock) -EXTRA_COUNTER_IDS = [ - 1255430, # 月 (months) - 1606800, # 割 (percentage) -] - -# Skip these counter IDs -SKIP_COUNTER_IDS = [ - 2426510, # 一個当り - 2220370, # 歳 (とせ) - 2248360, # 入 (しお) - 2423450, # 差し - 2671670, # 幅 (の) - 2735690, # 種 (くさ) - 2838543, # 杯 (はた) -] - -# Counters that accept certain suffixes -COUNTER_ACCEPTS = { - 1194480: ['kan'], # 年 - 1490430: ['kan'], # 日 - 1333450: ['kan', 'kango'], # 週 -} - -# Foreign-style counters (different phonetic rules) -COUNTER_FOREIGN = [1120410] - -# Counter suffixes (add to base counter) -COUNTER_SUFFIXES = { - 'kan': ('間', 'かん', '[duration]'), - 'kango': ('間後', 'かんご', '[after ...]'), - 'chuu': ('中', 'ちゅう', '[among/out of ...]'), -} - - -def get_counter_ids(session: Session) -> List[int]: - """Get all seq IDs that are counters.""" - result = session.execute( - select(SenseProp.seq) - .where(and_(SenseProp.tag == 'pos', SenseProp.text == 'ctr')) - .distinct() - ).scalars().all() - - return sorted(set(result) | set(EXTRA_COUNTER_IDS) - set(SKIP_COUNTER_IDS)) - - -def get_counter_readings(session: Session) -> Dict[int, Tuple[List[KanjiText], List[KanaText]]]: - """ - Get kanji and kana readings for all counters. - - Returns dict mapping seq to (kanji_list, kana_list). - """ - counter_ids = get_counter_ids(session) - if not counter_ids: - return {} - - result: Dict[int, Tuple[List[KanjiText], List[KanaText]]] = {} - - # Get kanji readings - kanji_readings = session.execute( - select(KanjiText).where(KanjiText.seq.in_(counter_ids)) - ).scalars().all() - - # Get kana readings - kana_readings = session.execute( - select(KanaText).where(KanaText.seq.in_(counter_ids)) - ).scalars().all() - - # Organize by seq - for kt in kanji_readings: - if kt.seq not in result: - result[kt.seq] = ([], []) - result[kt.seq][0].append(kt) - - for kt in kana_readings: - if kt.seq not in result: - result[kt.seq] = ([], []) - result[kt.seq][1].append(kt) - - # Sort by ord - for seq in result: - result[seq] = ( - sorted(result[seq][0], key=lambda x: x.ord), - sorted(result[seq][1], key=lambda x: x.ord) - ) - - return result - - -# ============================================================================ -# Phonetic Rules (Gemination/Rendaku) -# ============================================================================ - -def geminate(kana: str) -> str: - """ - Apply gemination (sokuon) to the end of a kana string. - E.g., "いち" -> "いっ" - """ - if not kana: - return kana - return kana[:-1] + 'っ' - - -def rendaku(kana: str, handakuten: bool = False) -> str: - """ - Apply rendaku (voicing) or handakuten to the beginning of a kana string. - E.g., "ひき" -> "びき" (rendaku) or "ぴき" (handakuten) - """ - if not kana: - return kana - - first_char = kana[0] - rest = kana[1:] - - if handakuten: - # h -> p - h_to_p = { - 'は': 'ぱ', 'ひ': 'ぴ', 'ふ': 'ぷ', 'へ': 'ぺ', 'ほ': 'ぽ', - 'ハ': 'パ', 'ヒ': 'ピ', 'フ': 'プ', 'ヘ': 'ペ', 'ホ': 'ポ', - } - if first_char in h_to_p: - return h_to_p[first_char] + rest - else: - # Voicing: k->g, s->z, t->d, h->b - voicing = { - 'か': 'が', 'き': 'ぎ', 'く': 'ぐ', 'け': 'げ', 'こ': 'ご', - 'さ': 'ざ', 'し': 'じ', 'す': 'ず', 'せ': 'ぜ', 'そ': 'ぞ', - 'た': 'だ', 'ち': 'ぢ', 'つ': 'づ', 'て': 'で', 'と': 'ど', - 'は': 'ば', 'ひ': 'び', 'ふ': 'ぶ', 'へ': 'べ', 'ほ': 'ぼ', - 'カ': 'ガ', 'キ': 'ギ', 'ク': 'グ', 'ケ': 'ゲ', 'コ': 'ゴ', - 'サ': 'ザ', 'シ': 'ジ', 'ス': 'ズ', 'セ': 'ゼ', 'ソ': 'ゾ', - 'タ': 'ダ', 'チ': 'ヂ', 'ツ': 'ヅ', 'テ': 'デ', 'ト': 'ド', - 'ハ': 'バ', 'ヒ': 'ビ', 'フ': 'ブ', 'ヘ': 'ベ', 'ホ': 'ボ', - } - if first_char in voicing: - return voicing[first_char] + rest - - return kana - - -def get_kana_head_class(kana: str) -> Optional[str]: - """Get the phonetic class of the first kana character.""" - if not kana: - return None - - first = kana[0] - - # Group by consonant - groups = { - 'ka': 'かきくけこカキクケコ', - 'sa': 'さしすせそサシスセソ', - 'ta': 'たちつてとタチツテト', - 'na': 'なにぬねのナニヌネノ', - 'ha': 'はひふへほハヒフヘホ', - 'ma': 'まみむめもマミムメモ', - 'ya': 'やゆよヤユヨ', - 'ra': 'らりるれろラリルレロ', - 'wa': 'わをんワヲン', - 'pa': 'ぱぴぷぺぽパピプペポ', - } - - for group, chars in groups.items(): - if first in chars: - return group - - return None - - -def counter_join(number_value: int, number_kana: str, counter_kana: str, - digit_opts: Optional[Dict] = None, foreign: bool = False) -> str: - """ - Join number kana with counter kana, applying phonetic rules. - - Ports ichiran's counter-join method from dict-counters.lisp. - - Args: - number_value: The numeric value - number_kana: The kana reading of the number - counter_kana: The kana reading of the counter - digit_opts: Optional dict mapping digits to phonetic options - foreign: Whether this is a foreign-style counter - - Returns: - The combined kana string with phonetic rules applied - """ - if digit_opts is None: - digit_opts = {} - - # Get the relevant digit - digit = number_value % 10 - if digit == 0: - # For powers of 10 - for power in [10, 100, 1000, 10000]: - if number_value % power == 0: - if number_value % (power * 10) != 0: - digit = power - break - - head = get_kana_head_class(counter_kana) - - # Check for explicit digit options - if digit in digit_opts: - opts = digit_opts[digit] - if isinstance(opts, str): - # Replace number reading with the string - # Get the stem length to replace - if digit < 10: - stem_kana = DIGIT_TO_KANA.get(digit, '') - else: - power = round(len(str(digit)) - 1) # log10 - stem_kana = POWER_TO_KANA.get(power, '') - - stem_len = len(stem_kana) if stem_kana else 0 - if stem_len > 0 and len(number_kana) >= stem_len: - return number_kana[:-stem_len] + opts + counter_kana - return opts + counter_kana - elif isinstance(opts, list): - # Apply modifications - result_number = number_kana - result_counter = counter_kana - for opt in opts: - if opt == 'g': - result_number = geminate(result_number) - elif opt == 'r': - result_counter = rendaku(result_counter) - elif opt == 'h': - result_counter = rendaku(result_counter, handakuten=True) - return result_number + result_counter - - # Apply standard rules based on digit and counter head - result_number = number_kana - result_counter = counter_kana - - if foreign: - # Simplified foreign counter rules - if digit in [6, 8, 10, 100] and head in ['ka', 'sa', 'ta', 'pa']: - result_number = geminate(result_number) - else: - # Japanese counter rules - if digit == 1: - if head in ['ka', 'sa', 'ta']: - result_number = geminate(result_number) - elif head == 'ha': - result_number = geminate(result_number) - result_counter = rendaku(result_counter, handakuten=True) - elif digit == 3: - if head == 'ha': - result_counter = rendaku(result_counter, handakuten=True) - elif digit == 6: - if head in ['ka', 'pa']: - result_number = geminate(result_number) - elif head == 'ha': - result_number = geminate(result_number) - result_counter = rendaku(result_counter, handakuten=True) - elif digit == 8: - if head in ['ka', 'sa', 'ta', 'pa']: - result_number = geminate(result_number) - elif head == 'ha': - result_number = geminate(result_number) - result_counter = rendaku(result_counter, handakuten=True) - elif digit == 10: - if head in ['ka', 'sa', 'ta', 'pa']: - result_number = geminate(result_number) - elif head == 'ha': - result_number = geminate(result_number) - result_counter = rendaku(result_counter, handakuten=True) - elif digit == 100: - if head == 'ka': - result_number = geminate(result_number) - elif head == 'ha': - result_number = geminate(result_number) - result_counter = rendaku(result_counter, handakuten=True) - elif digit in [1000, 10000]: - if head == 'ha': - result_counter = rendaku(result_counter, handakuten=True) - - return result_number + result_counter - - -# ============================================================================ -# Counter Lookup -# ============================================================================ - -# Special counter kana overrides -# Maps seq to the kana reading to use for the counter -SPECIAL_COUNTER_KANA_OVERRIDE: Dict[int, str] = { - 1255430: 'がつ', # 月 (months) - uses がつ not つき -} - -def init_counter_cache(session: Session) -> None: - """Initialize the counter cache with all counter readings.""" - global _counter_cache, _counter_cache_initialized - - if _counter_cache_initialized: - return - - _counter_cache = {} - counter_readings = get_counter_readings(session) - - for seq, (kanji_list, kana_list) in counter_readings.items(): - if not kana_list: - continue - - # Primary kana reading - check for override first - if seq in SPECIAL_COUNTER_KANA_OVERRIDE: - primary_kana = SPECIAL_COUNTER_KANA_OVERRIDE[seq] - else: - primary_kana = kana_list[0].text if kana_list else '' - - # Get digit_opts for this counter if it's a special counter - digit_opts = SPECIAL_COUNTER_OPTS.get(seq) - - # Use kanji readings as counter text - for kt in kanji_list: - counter_text = kt.text - is_ordinal = len(counter_text) > 1 and counter_text.endswith('目') - - # Add to cache - if counter_text not in _counter_cache: - _counter_cache[counter_text] = [] - - _counter_cache[counter_text].append({ - 'class': CounterText, - 'counter_text': counter_text, - 'counter_kana': primary_kana, - 'source': kt, - 'ordinalp': is_ordinal, - 'common': kt.common, - 'digit_opts': digit_opts, - }) - - _counter_cache_initialized = True - - -def find_counter( - session: Session, - number_text: str, - counter_text: str, - unique: bool = True, -) -> List[CounterText]: - """ - Find counter expressions matching number + counter. - - Args: - session: Database session - number_text: Number part (e.g., "三") - counter_text: Counter part (e.g., "匹") - unique: If True, only return verified counters - - Returns: - List of CounterText objects - """ - init_counter_cache(session) - - # Parse the number - number_value = parse_number(number_text) - if number_value is None: - return [] - - # Look up the counter - counter_args_list = _counter_cache.get(counter_text, []) - if not counter_args_list: - return [] - - results = [] - for args in counter_args_list: - source = args.get('source') - seq = source.seq if source else None - digit_opts = args.get('digit_opts') - counter_kana = args['counter_kana'] - - # Check for special kana readings - special_kana = None - common_override = args.get('common') - - # Days counter (日 ka) - seq 2083110 - # Only valid for 1-10, 14, 20, 24, 30 - if seq == 2083110: - if number_value in DAYS_KUN_READINGS: - special_kana = DAYS_KUN_READINGS[number_value] - # Use source common value, don't override - else: - # Skip this counter for numbers without kun readings - continue - - # Days counter (日 nichi) - seq 2083100 - # Valid for numbers > 10 (except 14, 20, 24, 30) and 1 - if seq == 2083100: - if number_value in DAYS_KUN_READINGS and number_value != 1: - # Skip nichi reading for numbers that have kun readings (except 1) - continue - - # People counter (人 nin) - seq 2149890 - if seq == 2149890: - if number_value in PEOPLE_KUN_READINGS: - special_kana = PEOPLE_KUN_READINGS[number_value] - - # Generate the kana reading - if special_kana: - full_kana = special_kana - else: - number_kana = number_to_kana(number_value) - # Apply phonetic rules with digit_opts - full_kana = counter_join(number_value, number_kana, counter_kana, digit_opts=digit_opts) - - full_text = number_text + counter_text - - counter_obj = CounterText( - text=full_text, - kana=full_kana, - number_text=number_text, - number_value=number_value, - counter_text=counter_text, - counter_kana=counter_kana, - source=source, - ordinalp=args.get('ordinalp', False), - _common=common_override, - digit_opts=digit_opts, - ) - - results.append(counter_obj) - - return results - - -def find_counter_in_text( - session: Session, - text: str, -) -> List[Tuple[int, int, CounterText]]: - """ - Find all counter expressions in a text. - - Returns list of (start, end, CounterText) tuples. - """ - init_counter_cache(session) - - results = [] - text_len = len(text) - - for start in range(text_len): - # Find number prefix - number_end = start - while number_end < text_len: - char = text[number_end] - if char in KANJI_NUMBERS or char in DIGIT_VALUES or char.isdigit(): - number_end += 1 - else: - break - - if number_end == start: - continue - - number_text = text[start:number_end] - number_value = parse_number(number_text) - if number_value is None: - continue - - # Try to find matching counters - for end in range(number_end + 1, min(text_len + 1, number_end + 5)): - counter_text = text[number_end:end] - counters = find_counter(session, number_text, counter_text) - - for counter in counters: - results.append((start, end, counter)) - - return results - - -# ============================================================================ -# Counter Scoring -# ============================================================================ - -def calc_counter_score(counter: CounterText, use_length: Optional[int] = None) -> int: - """ - Calculate score for a counter expression. - - Counter expressions get a base score of at least 5 (like kanji words). - """ - # Base score similar to regular words - score = 5 # Minimum like kanji words - - # Common counter bonus - if counter.common is not None and counter.common == 0: - score += 10 - elif counter.common is not None: - score += max(15 - counter.common, 5) - - # Length bonus - word_len = mora_length(counter.text) - - # Use strong length coefficients (like kanji) - # strong: [0, 1, 8, 24, 40, 60] - length_coeffs = [0, 1, 8, 24, 40, 60] - if word_len < len(length_coeffs): - coeff = length_coeffs[word_len] - else: - coeff = word_len * (length_coeffs[-1] // (len(length_coeffs) - 1)) - - final_score = score * coeff - - return final_score +"""Backward-compatible shim for himotoki.counters.""" +from himotoki.grammar.counters import * # noqa: F401,F403 diff --git a/himotoki/db/connection.py b/himotoki/db/connection.py index 709ec9c..18a9731 100644 --- a/himotoki/db/connection.py +++ b/himotoki/db/connection.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Optional, Generator -from sqlalchemy import create_engine, event +from sqlalchemy import create_engine, event, text from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.pool import StaticPool @@ -235,6 +235,11 @@ def init_database(db_path: Optional[str] = None, drop_existing: bool = False): if db_path is not None: _engine = None _session_factory = None + try: + from himotoki.scoring.caches import clear_scoring_caches + clear_scoring_caches() + except Exception: + pass engine = get_engine(db_path) @@ -263,6 +268,13 @@ def close_connection(): with _cache_lock: _cache.clear() + # Scoring caches hold ORM objects / seq sets tied to the open DB + try: + from himotoki.scoring.caches import clear_scoring_caches + clear_scoring_caches() + except Exception: + pass + # Cache management functions (similar to ichiran's defcache) @@ -341,7 +353,8 @@ def analyze(): """ engine = get_engine() with engine.connect() as conn: - conn.execute("ANALYZE") + conn.execute(text("ANALYZE")) + conn.commit() def vacuum(): @@ -349,5 +362,5 @@ def vacuum(): Run VACUUM on the database to reclaim space. """ engine = get_engine() - with engine.connect() as conn: - conn.execute("VACUUM") \ No newline at end of file + with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: + conn.execute(text("VACUUM")) \ No newline at end of file diff --git a/himotoki/db/models.py b/himotoki/db/models.py index a5b7ce9..b2b3760 100644 --- a/himotoki/db/models.py +++ b/himotoki/db/models.py @@ -26,7 +26,7 @@ class Entry(Base): __tablename__ = "entry" seq = Column(Integer, primary_key=True) - content = Column(Text, nullable=False, default="") # Original XML content + content = Column(Text, nullable=False, default="") # Unused (kept for schema compatibility) root_p = Column(Boolean, nullable=False, default=False) # True if this is a root entry (not conjugation) n_kanji = Column(Integer, nullable=False, default=0) # Number of kanji readings n_kana = Column(Integer, nullable=False, default=0) # Number of kana readings @@ -77,13 +77,9 @@ class KanjiText(Base): entry = relationship("Entry", back_populates="kanji_texts") __table_args__ = ( - Index("ix_kanji_text_seq", "seq"), - Index("ix_kanji_text_ord", "ord"), - Index("ix_kanji_text_text", "text"), - Index("ix_kanji_text_common", "common"), - # Composite indexes for common query patterns - Index("ix_kanji_text_text_seq", "text", "seq"), # find_word lookups - Index("ix_kanji_text_seq_ord", "seq", "ord"), # ordered retrieval by entry + # Covering index for hot-path: SELECT id,seq,text,ord,common,best_kana WHERE text IN (...) + Index("ix_kanji_text_text_cover", "text", "seq", "id", "ord", "common", "best_kana"), + Index("ix_kanji_text_seq_ord", "seq", "ord"), ) def __repr__(self): @@ -111,17 +107,12 @@ class KanaText(Base): entry = relationship("Entry", back_populates="kana_texts") __table_args__ = ( - Index("ix_kana_text_seq", "seq"), - Index("ix_kana_text_ord", "ord"), - Index("ix_kana_text_text", "text"), - Index("ix_kana_text_common", "common"), - # Composite indexes for common query patterns - Index("ix_kana_text_text_seq", "text", "seq"), # find_word lookups - Index("ix_kana_text_seq_ord", "seq", "ord"), # ordered retrieval by entry + Index("ix_kana_text_text_cover", "text", "seq", "id", "ord", "common", "best_kanji"), + Index("ix_kana_text_seq_ord", "seq", "ord"), ) def __repr__(self): - return f"" class Sense(Base): @@ -293,7 +284,7 @@ class ConjSourceReading(Base): conjugation = relationship("Conjugation", back_populates="source_readings") __table_args__ = ( - Index("ix_conj_source_reading_conj_id_text", "conj_id", "text"), + Index("ix_conj_source_reading_conj_id", "conj_id"), ) def __repr__(self): diff --git a/himotoki/grammar/__init__.py b/himotoki/grammar/__init__.py new file mode 100644 index 0000000..b8910c9 --- /dev/null +++ b/himotoki/grammar/__init__.py @@ -0,0 +1,40 @@ +""" +Grammar rules: synergies, suffixes, splits, counters. +""" + +from himotoki.grammar.counters import * # noqa: F401,F403 +from himotoki.grammar.splits import * # noqa: F401,F403 +from himotoki.grammar.synergies import ( # noqa: F401 + Synergy, + get_synergies, + get_penalties, + apply_segfilters, + get_segment_score_synergy, + register_synergy, + register_penalty, + register_segfilter, + def_generic_synergy, + def_generic_penalty, + def_segfilter_must_follow, +) +from himotoki.grammar.synergy_filters import ( # noqa: F401 + filter_is_noun, + filter_is_pos, + filter_in_seq_set, + filter_in_seq_set_simple, + filter_is_conjugation, + filter_is_compound_end, + filter_is_compound_end_text, + filter_short_kana, +) +from himotoki.grammar.suffixes import ( # noqa: F401 + init_suffixes, + is_suffix_cache_ready, + find_word_suffix, + could_have_suffix, + get_suffix_description, + SUFFIX_SCORES, + SUFFIX_CONNECTORS, +) +from himotoki.constants import NOUN_PARTICLES # noqa: F401 + diff --git a/himotoki/grammar/counters.py b/himotoki/grammar/counters.py new file mode 100644 index 0000000..7539703 --- /dev/null +++ b/himotoki/grammar/counters.py @@ -0,0 +1,914 @@ +""" +Counter word recognition module for himotoki. +Ports ichiran's dict-counters.lisp counter functionality. + +Counter expressions combine a number with a counter suffix to form compound words. +Examples: +- 三匹 (sanbiki) = 三 (san, three) + 匹 (hiki, counter for small animals) +- 五冊 (gosatsu) = 五 (go, five) + 冊 (satsu, counter for books) +- 千人 (sennin) = 千 (sen, thousand) + 人 (nin, counter for people) + +This module dynamically generates counter expressions and provides scoring. +""" + +from dataclasses import dataclass, field +from typing import Optional, List, Dict, Tuple, Any, Set, Union +from functools import lru_cache + +from sqlalchemy import select, and_ +from sqlalchemy.orm import Session + +from himotoki.db.models import ( + Entry, KanjiText, KanaText, SenseProp, +) +from himotoki.characters import ( + get_char_class, mora_length, as_hiragana, count_char_class, +) + + +# ============================================================================ +# Japanese Number Parsing +# ============================================================================ + +# Number kanji to value mapping +KANJI_NUMBERS = { + '零': 0, '〇': 0, + '一': 1, '壱': 1, + '二': 2, '弐': 2, + '三': 3, '参': 3, + '四': 4, + '五': 5, + '六': 6, + '七': 7, + '八': 8, + '九': 9, + '十': 10, + '百': 100, + '千': 1000, + '万': 10000, + '億': 100000000, +} + +# Arabic digit to value +DIGIT_VALUES = { + '0': 0, '１': 0, '０': 0, + '1': 1, '１': 1, + '2': 2, '２': 2, + '3': 3, '３': 3, + '4': 4, '４': 4, + '5': 5, '５': 5, + '6': 6, '６': 6, + '7': 7, '７': 7, + '8': 8, '８': 8, + '9': 9, '９': 9, +} + +# Number kana to digit +DIGIT_TO_KANA = { + 1: 'いち', + 2: 'に', + 3: 'さん', + 4: 'よん', # or し + 5: 'ご', + 6: 'ろく', + 7: 'なな', # or しち + 8: 'はち', + 9: 'きゅう', # or く + 10: 'じゅう', + 100: 'ひゃく', + 1000: 'せん', + 10000: 'まん', +} + +# Power values to kana +POWER_TO_KANA = { + 1: 'じゅう', # 10 + 2: 'ひゃく', # 100 + 3: 'せん', # 1000 + 4: 'まん', # 10000 +} + + +def parse_number(text: str) -> Optional[int]: + """ + Parse a Japanese number string into an integer. + + Supports: + - Arabic numerals (1, 2, 3 or １, ２, ３) + - Kanji numerals (一, 二, 三) + - Mixed (十二, 百三十四) + - Arabic + Kanji multiplier (5億, 10万) + + Returns None if the text is not a valid number. + """ + if not text: + return None + + # Try Arabic numerals first + arabic_str = '' + for char in text: + if char in DIGIT_VALUES: + arabic_str += str(DIGIT_VALUES[char]) + elif char.isdigit(): + arabic_str += char + else: + break + + if arabic_str and len(arabic_str) == len(text): + return int(arabic_str) + + # Try Arabic prefix + Kanji multiplier (e.g., 5億, 10万) + if arabic_str: + arabic_value = int(arabic_str) + suffix = text[len(arabic_str):] + + # Check if suffix is a single multiplier + if len(suffix) == 1 and suffix in KANJI_NUMBERS: + multiplier = KANJI_NUMBERS[suffix] + if multiplier >= 10: # 十, 百, 千, 万, 億 + return arabic_value * multiplier + + # Check if suffix is multiple multipliers (e.g., 10万億 -> 10 * 万 * 億) + # Not implemented for now, but could extend if needed + + # Try kanji numerals + return parse_kanji_number(text) + + +def parse_kanji_number(text: str) -> Optional[int]: + """Parse a kanji number string into an integer.""" + if not text: + return None + + result = 0 + current = 0 + + for char in text: + if char not in KANJI_NUMBERS: + return None + + value = KANJI_NUMBERS[char] + + if value >= 10: + # Multiplier (十, 百, 千, 万, 億) + if current == 0: + current = 1 + + if value >= 10000: + # 万 or higher - adds to result + result += current * value + current = 0 + else: + # 十, 百, 千 - multiplies current + current *= value + result += current + current = 0 + else: + # Digit + if current > 0: + result += current + current = value + + # Add any remaining current + result += current + + return result if result > 0 or text == '零' or text == '〇' else None + + +def number_to_kana(n: int, separator: str = '') -> str: + """Convert a number to its kana reading.""" + if n == 0: + return 'ゼロ' + + parts = [] + + # Handle 100000000s (億) + if n >= 100000000: + oku = n // 100000000 + if oku > 1: + parts.append(number_to_kana(oku)) + else: + parts.append('いち') + parts.append('おく') + n %= 100000000 + + # Handle 10000s (万) + if n >= 10000: + man = n // 10000 + if man > 1: + parts.append(number_to_kana(man)) + parts.append('まん') + n %= 10000 + + # Handle 1000s + if n >= 1000: + sen = n // 1000 + if sen == 3: + parts.append('さん') + elif sen > 1: + parts.append(DIGIT_TO_KANA.get(sen, '')) + parts.append('せん') + n %= 1000 + + # Handle 100s + if n >= 100: + hyaku = n // 100 + if hyaku == 3: + parts.append('さん') + parts.append('びゃく') # 300 = sanbyaku + elif hyaku == 6: + parts.append('ろっ') + parts.append('ぴゃく') # 600 = roppyaku + elif hyaku == 8: + parts.append('はっ') + parts.append('ぴゃく') # 800 = happyaku + else: + if hyaku > 1: + parts.append(DIGIT_TO_KANA.get(hyaku, '')) + parts.append('ひゃく') + n %= 100 + + # Handle 10s + if n >= 10: + juu = n // 10 + if juu > 1: + parts.append(DIGIT_TO_KANA.get(juu, '')) + parts.append('じゅう') + n %= 10 + + # Handle 1s + if n > 0: + parts.append(DIGIT_TO_KANA.get(n, '')) + + return separator.join(parts) + + +# ============================================================================ +# Counter Text Class +# ============================================================================ + +@dataclass +class CounterText: + """ + Represents a counter expression (number + counter). + + Ports ichiran's counter-text class from dict-counters.lisp. + + CounterText objects are designed to be compatible with WordMatch + so they can be scored using the main calc_score function. + """ + text: str # Full text (e.g., "三匹") + kana: str # Kana reading (e.g., "さんびき") + number_text: str # Number part (e.g., "三") + number_value: int # Numeric value (e.g., 3) + counter_text: str # Counter part (e.g., "匹") + counter_kana: str # Counter kana (e.g., "ひき") + source: Optional[Union[KanjiText, KanaText]] = None # Source counter entry + ordinalp: bool = False # Is this ordinal? (〜目) + suffix: Optional[str] = None # Kana suffix (e.g., "め" for 目) + _common: Optional[int] = None # Commonness rating (internal) + digit_opts: Optional[Dict] = None # Digit-specific options for phonetic rules + + @property + def seq(self) -> Optional[int]: + """Get the seq from the source counter.""" + return self.source.seq if self.source else None + + @property + def ord(self) -> int: + """Get the ord from the source counter.""" + return self.source.ord if self.source else 0 + + @property + def common(self) -> Optional[int]: + """Get commonness rating from source or override.""" + if self._common is not None: + return self._common + if self.source: + return self.source.common + return 0 + + @property + def word_type(self) -> str: + """Counter expressions are typically kanji.""" + # Check if text contains kanji + if count_char_class(self.text, 'kanji') > 0: + return 'kanji' + return 'kana' + + @property + def reading(self): + """Return self as the reading for compatibility with calc_score.""" + return self + + @property + def conjugations(self) -> None: + """Counters don't have conjugations.""" + return None + + @property + def is_root(self) -> bool: + """Counters are always root forms.""" + return True + + @property + def is_compound(self) -> bool: + """Counters are not compound words.""" + return False + + @property + def components(self) -> List[str]: + """Return empty list for counters (no components).""" + return [] + + def __repr__(self): + return f"" + + +# ============================================================================ +# Counter Cache +# ============================================================================ + +# Cache for counter readings +_counter_cache: Dict[str, List[Tuple[type, Dict]]] = {} +_counter_cache_initialized = False + +# Special counter handling overrides +# Maps seq to digit_opts dict for phonetic rules +# Format: {digit: [opts]} where opts can be 'g' (geminate), 'r' (rendaku), 'h' (handakuten) +# or a string to replace the number reading +SPECIAL_COUNTER_OPTS: Dict[int, Dict] = { + # 匹/疋 (hiki) - counter for small animals + 1583370: {3: ['r']}, # 3 -> さんびき (rendaku) + # 本 (hon) - counter for long thin objects + 1522150: {3: ['r']}, # 3 -> さんぼん (rendaku) + # 杯/盃 (hai) - counter for cups/glasses + 2019640: {3: ['r']}, # 3 -> さんばい (rendaku) + # 階 (kai) - counter for floors + 1203020: {3: ['r']}, # 3 -> さんがい (rendaku) + # 軒 (ken) - counter for houses + 2078590: {3: ['r']}, # 3 -> さんげん (rendaku) + # 遍 (hen) - counter for times + 2208060: {3: ['r']}, # 3 -> さんべん (rendaku) + # 編/篇 (hen) - counter for literary works + 1511870: {3: ['r']}, # 3 -> さんべん (rendaku) + # 足 (soku) - counter for pairs of footwear + 2412230: {3: ['r']}, # 3 -> さんぞく (rendaku) + # 時 (ji) - counter for hours + 2020680: {4: 'よ', 7: 'しち', 9: 'く'}, + # 時間 (jikan) - counter for hours duration + 1315920: {4: 'よ', 9: 'く'}, + # 年 (nen) - counter for years + 2084840: {4: 'よ', 7: 'しち', 9: 'く'}, + # 円 (en) - counter for yen + 1175570: {4: 'よ'}, + # 分 (fun) - counter for minutes + 1502840: {4: ['h']}, # 4 -> よんぷん (handakuten) + # 舗 (ho) - counter for shops + 1514050: {4: ['h']}, + # 敗 (hai) - counter for losses + 1901390: {4: ['h']}, + # 泊 (haku) - counter for nights stay + 1919550: {4: ['h']}, + # 筆 (hitsu) - counter for brush strokes + 1487770: {4: ['h']}, + # 冊 (satsu) - counter for books - uses standard rules + # 人 (nin) - counter for people - special class + 2149890: {4: 'よ', 7: 'しち'}, + # 月 (gatsu) - counter for months + 1255430: {4: 'し', 7: 'しち', 9: 'く'}, +} + +# Special counter classes that need custom kana generation +# Maps seq to a function that returns the kana for a given number +SPECIAL_COUNTER_KANA: Dict[int, callable] = {} + +# Days counter (日 ka) - kun readings for 1-10, 14, 20, 24, 30 +DAYS_KUN_READINGS = { + 1: 'ついたち', + 2: 'ふつか', + 3: 'みっか', + 4: 'よっか', + 5: 'いつか', + 6: 'むいか', + 7: 'なのか', + 8: 'ようか', + 9: 'ここのか', + 10: 'とうか', + 14: 'じゅうよっか', + 20: 'はつか', + 24: 'にじゅうよっか', + 30: 'みそか', +} + +# People counter (人 nin) - kun readings for 1-2 +PEOPLE_KUN_READINGS = { + 1: 'ひとり', + 2: 'ふたり', +} + +SPECIAL_COUNTERS: Dict[int, callable] = {} + +# Extra counter IDs (not marked as ctr but act as counters) +# Note: 時 (2020680) is intentionally not included because it conflicts with +# regular dictionary entries like 三時 (3 o'clock) +EXTRA_COUNTER_IDS = [ + 1255430, # 月 (months) + 1606800, # 割 (percentage) +] + +# Skip these counter IDs +SKIP_COUNTER_IDS = [ + 2426510, # 一個当り + 2220370, # 歳 (とせ) + 2248360, # 入 (しお) + 2423450, # 差し + 2671670, # 幅 (の) + 2735690, # 種 (くさ) + 2838543, # 杯 (はた) +] + +# Counters that accept certain suffixes +COUNTER_ACCEPTS = { + 1194480: ['kan'], # 年 + 1490430: ['kan'], # 日 + 1333450: ['kan', 'kango'], # 週 +} + +# Foreign-style counters (different phonetic rules) +COUNTER_FOREIGN = [1120410] + +# Counter suffixes (add to base counter) +COUNTER_SUFFIXES = { + 'kan': ('間', 'かん', '[duration]'), + 'kango': ('間後', 'かんご', '[after ...]'), + 'chuu': ('中', 'ちゅう', '[among/out of ...]'), +} + + +def get_counter_ids(session: Session) -> List[int]: + """Get all seq IDs that are counters.""" + result = session.execute( + select(SenseProp.seq) + .where(and_(SenseProp.tag == 'pos', SenseProp.text == 'ctr')) + .distinct() + ).scalars().all() + + return sorted(set(result) | set(EXTRA_COUNTER_IDS) - set(SKIP_COUNTER_IDS)) + + +def get_counter_readings(session: Session) -> Dict[int, Tuple[List[KanjiText], List[KanaText]]]: + """ + Get kanji and kana readings for all counters. + + Returns dict mapping seq to (kanji_list, kana_list). + """ + counter_ids = get_counter_ids(session) + if not counter_ids: + return {} + + result: Dict[int, Tuple[List[KanjiText], List[KanaText]]] = {} + + # Get kanji readings + kanji_readings = session.execute( + select(KanjiText).where(KanjiText.seq.in_(counter_ids)) + ).scalars().all() + + # Get kana readings + kana_readings = session.execute( + select(KanaText).where(KanaText.seq.in_(counter_ids)) + ).scalars().all() + + # Organize by seq + for kt in kanji_readings: + if kt.seq not in result: + result[kt.seq] = ([], []) + result[kt.seq][0].append(kt) + + for kt in kana_readings: + if kt.seq not in result: + result[kt.seq] = ([], []) + result[kt.seq][1].append(kt) + + # Sort by ord + for seq in result: + result[seq] = ( + sorted(result[seq][0], key=lambda x: x.ord), + sorted(result[seq][1], key=lambda x: x.ord) + ) + + return result + + +# ============================================================================ +# Phonetic Rules (Gemination/Rendaku) +# ============================================================================ + +def geminate(kana: str) -> str: + """ + Apply gemination (sokuon) to the end of a kana string. + E.g., "いち" -> "いっ" + """ + if not kana: + return kana + return kana[:-1] + 'っ' + + +def rendaku(kana: str, handakuten: bool = False) -> str: + """ + Apply rendaku (voicing) or handakuten to the beginning of a kana string. + E.g., "ひき" -> "びき" (rendaku) or "ぴき" (handakuten) + """ + if not kana: + return kana + + first_char = kana[0] + rest = kana[1:] + + if handakuten: + # h -> p + h_to_p = { + 'は': 'ぱ', 'ひ': 'ぴ', 'ふ': 'ぷ', 'へ': 'ぺ', 'ほ': 'ぽ', + 'ハ': 'パ', 'ヒ': 'ピ', 'フ': 'プ', 'ヘ': 'ペ', 'ホ': 'ポ', + } + if first_char in h_to_p: + return h_to_p[first_char] + rest + else: + # Voicing: k->g, s->z, t->d, h->b + voicing = { + 'か': 'が', 'き': 'ぎ', 'く': 'ぐ', 'け': 'げ', 'こ': 'ご', + 'さ': 'ざ', 'し': 'じ', 'す': 'ず', 'せ': 'ぜ', 'そ': 'ぞ', + 'た': 'だ', 'ち': 'ぢ', 'つ': 'づ', 'て': 'で', 'と': 'ど', + 'は': 'ば', 'ひ': 'び', 'ふ': 'ぶ', 'へ': 'べ', 'ほ': 'ぼ', + 'カ': 'ガ', 'キ': 'ギ', 'ク': 'グ', 'ケ': 'ゲ', 'コ': 'ゴ', + 'サ': 'ザ', 'シ': 'ジ', 'ス': 'ズ', 'セ': 'ゼ', 'ソ': 'ゾ', + 'タ': 'ダ', 'チ': 'ヂ', 'ツ': 'ヅ', 'テ': 'デ', 'ト': 'ド', + 'ハ': 'バ', 'ヒ': 'ビ', 'フ': 'ブ', 'ヘ': 'ベ', 'ホ': 'ボ', + } + if first_char in voicing: + return voicing[first_char] + rest + + return kana + + +def get_kana_head_class(kana: str) -> Optional[str]: + """Get the phonetic class of the first kana character.""" + if not kana: + return None + + first = kana[0] + + # Group by consonant + groups = { + 'ka': 'かきくけこカキクケコ', + 'sa': 'さしすせそサシスセソ', + 'ta': 'たちつてとタチツテト', + 'na': 'なにぬねのナニヌネノ', + 'ha': 'はひふへほハヒフヘホ', + 'ma': 'まみむめもマミムメモ', + 'ya': 'やゆよヤユヨ', + 'ra': 'らりるれろラリルレロ', + 'wa': 'わをんワヲン', + 'pa': 'ぱぴぷぺぽパピプペポ', + } + + for group, chars in groups.items(): + if first in chars: + return group + + return None + + +def counter_join(number_value: int, number_kana: str, counter_kana: str, + digit_opts: Optional[Dict] = None, foreign: bool = False) -> str: + """ + Join number kana with counter kana, applying phonetic rules. + + Ports ichiran's counter-join method from dict-counters.lisp. + + Args: + number_value: The numeric value + number_kana: The kana reading of the number + counter_kana: The kana reading of the counter + digit_opts: Optional dict mapping digits to phonetic options + foreign: Whether this is a foreign-style counter + + Returns: + The combined kana string with phonetic rules applied + """ + if digit_opts is None: + digit_opts = {} + + # Get the relevant digit + digit = number_value % 10 + if digit == 0: + # For powers of 10 + for power in [10, 100, 1000, 10000]: + if number_value % power == 0: + if number_value % (power * 10) != 0: + digit = power + break + + head = get_kana_head_class(counter_kana) + + # Check for explicit digit options + if digit in digit_opts: + opts = digit_opts[digit] + if isinstance(opts, str): + # Replace number reading with the string + # Get the stem length to replace + if digit < 10: + stem_kana = DIGIT_TO_KANA.get(digit, '') + else: + power = round(len(str(digit)) - 1) # log10 + stem_kana = POWER_TO_KANA.get(power, '') + + stem_len = len(stem_kana) if stem_kana else 0 + if stem_len > 0 and len(number_kana) >= stem_len: + return number_kana[:-stem_len] + opts + counter_kana + return opts + counter_kana + elif isinstance(opts, list): + # Apply modifications + result_number = number_kana + result_counter = counter_kana + for opt in opts: + if opt == 'g': + result_number = geminate(result_number) + elif opt == 'r': + result_counter = rendaku(result_counter) + elif opt == 'h': + result_counter = rendaku(result_counter, handakuten=True) + return result_number + result_counter + + # Apply standard rules based on digit and counter head + result_number = number_kana + result_counter = counter_kana + + if foreign: + # Simplified foreign counter rules + if digit in [6, 8, 10, 100] and head in ['ka', 'sa', 'ta', 'pa']: + result_number = geminate(result_number) + else: + # Japanese counter rules + if digit == 1: + if head in ['ka', 'sa', 'ta']: + result_number = geminate(result_number) + elif head == 'ha': + result_number = geminate(result_number) + result_counter = rendaku(result_counter, handakuten=True) + elif digit == 3: + if head == 'ha': + result_counter = rendaku(result_counter, handakuten=True) + elif digit == 6: + if head in ['ka', 'pa']: + result_number = geminate(result_number) + elif head == 'ha': + result_number = geminate(result_number) + result_counter = rendaku(result_counter, handakuten=True) + elif digit == 8: + if head in ['ka', 'sa', 'ta', 'pa']: + result_number = geminate(result_number) + elif head == 'ha': + result_number = geminate(result_number) + result_counter = rendaku(result_counter, handakuten=True) + elif digit == 10: + if head in ['ka', 'sa', 'ta', 'pa']: + result_number = geminate(result_number) + elif head == 'ha': + result_number = geminate(result_number) + result_counter = rendaku(result_counter, handakuten=True) + elif digit == 100: + if head == 'ka': + result_number = geminate(result_number) + elif head == 'ha': + result_number = geminate(result_number) + result_counter = rendaku(result_counter, handakuten=True) + elif digit in [1000, 10000]: + if head == 'ha': + result_counter = rendaku(result_counter, handakuten=True) + + return result_number + result_counter + + +# ============================================================================ +# Counter Lookup +# ============================================================================ + +# Special counter kana overrides +# Maps seq to the kana reading to use for the counter +SPECIAL_COUNTER_KANA_OVERRIDE: Dict[int, str] = { + 1255430: 'がつ', # 月 (months) - uses がつ not つき +} + +def init_counter_cache(session: Session) -> None: + """Initialize the counter cache with all counter readings.""" + global _counter_cache, _counter_cache_initialized + + if _counter_cache_initialized: + return + + _counter_cache = {} + counter_readings = get_counter_readings(session) + + for seq, (kanji_list, kana_list) in counter_readings.items(): + if not kana_list: + continue + + # Primary kana reading - check for override first + if seq in SPECIAL_COUNTER_KANA_OVERRIDE: + primary_kana = SPECIAL_COUNTER_KANA_OVERRIDE[seq] + else: + primary_kana = kana_list[0].text if kana_list else '' + + # Get digit_opts for this counter if it's a special counter + digit_opts = SPECIAL_COUNTER_OPTS.get(seq) + + # Use kanji readings as counter text + for kt in kanji_list: + counter_text = kt.text + is_ordinal = len(counter_text) > 1 and counter_text.endswith('目') + + # Add to cache + if counter_text not in _counter_cache: + _counter_cache[counter_text] = [] + + _counter_cache[counter_text].append({ + 'class': CounterText, + 'counter_text': counter_text, + 'counter_kana': primary_kana, + 'source': kt, + 'ordinalp': is_ordinal, + 'common': kt.common, + 'digit_opts': digit_opts, + }) + + _counter_cache_initialized = True + + +def find_counter( + session: Session, + number_text: str, + counter_text: str, + unique: bool = True, +) -> List[CounterText]: + """ + Find counter expressions matching number + counter. + + Args: + session: Database session + number_text: Number part (e.g., "三") + counter_text: Counter part (e.g., "匹") + unique: If True, only return verified counters + + Returns: + List of CounterText objects + """ + init_counter_cache(session) + + # Parse the number + number_value = parse_number(number_text) + if number_value is None: + return [] + + # Look up the counter + counter_args_list = _counter_cache.get(counter_text, []) + if not counter_args_list: + return [] + + results = [] + for args in counter_args_list: + source = args.get('source') + seq = source.seq if source else None + digit_opts = args.get('digit_opts') + counter_kana = args['counter_kana'] + + # Check for special kana readings + special_kana = None + common_override = args.get('common') + + # Days counter (日 ka) - seq 2083110 + # Only valid for 1-10, 14, 20, 24, 30 + if seq == 2083110: + if number_value in DAYS_KUN_READINGS: + special_kana = DAYS_KUN_READINGS[number_value] + # Use source common value, don't override + else: + # Skip this counter for numbers without kun readings + continue + + # Days counter (日 nichi) - seq 2083100 + # Valid for numbers > 10 (except 14, 20, 24, 30) and 1 + if seq == 2083100: + if number_value in DAYS_KUN_READINGS and number_value != 1: + # Skip nichi reading for numbers that have kun readings (except 1) + continue + + # People counter (人 nin) - seq 2149890 + if seq == 2149890: + if number_value in PEOPLE_KUN_READINGS: + special_kana = PEOPLE_KUN_READINGS[number_value] + + # Generate the kana reading + if special_kana: + full_kana = special_kana + else: + number_kana = number_to_kana(number_value) + # Apply phonetic rules with digit_opts + full_kana = counter_join(number_value, number_kana, counter_kana, digit_opts=digit_opts) + + full_text = number_text + counter_text + + counter_obj = CounterText( + text=full_text, + kana=full_kana, + number_text=number_text, + number_value=number_value, + counter_text=counter_text, + counter_kana=counter_kana, + source=source, + ordinalp=args.get('ordinalp', False), + _common=common_override, + digit_opts=digit_opts, + ) + + results.append(counter_obj) + + return results + + +def find_counter_in_text( + session: Session, + text: str, +) -> List[Tuple[int, int, CounterText]]: + """ + Find all counter expressions in a text. + + Returns list of (start, end, CounterText) tuples. + """ + init_counter_cache(session) + + results = [] + text_len = len(text) + + for start in range(text_len): + # Find number prefix + number_end = start + while number_end < text_len: + char = text[number_end] + if char in KANJI_NUMBERS or char in DIGIT_VALUES or char.isdigit(): + number_end += 1 + else: + break + + if number_end == start: + continue + + number_text = text[start:number_end] + number_value = parse_number(number_text) + if number_value is None: + continue + + # Try to find matching counters + for end in range(number_end + 1, min(text_len + 1, number_end + 5)): + counter_text = text[number_end:end] + counters = find_counter(session, number_text, counter_text) + + for counter in counters: + results.append((start, end, counter)) + + return results + + +# ============================================================================ +# Counter Scoring +# ============================================================================ + +def calc_counter_score(counter: CounterText, use_length: Optional[int] = None) -> int: + """ + Calculate score for a counter expression. + + Counter expressions get a base score of at least 5 (like kanji words). + """ + # Base score similar to regular words + score = 5 # Minimum like kanji words + + # Common counter bonus + if counter.common is not None and counter.common == 0: + score += 10 + elif counter.common is not None: + score += max(15 - counter.common, 5) + + # Length bonus + word_len = mora_length(counter.text) + + # Use strong length coefficients (like kanji) + # strong: [0, 1, 8, 24, 40, 60] + length_coeffs = [0, 1, 8, 24, 40, 60] + if word_len < len(length_coeffs): + coeff = length_coeffs[word_len] + else: + coeff = word_len * (length_coeffs[-1] // (len(length_coeffs) - 1)) + + final_score = score * coeff + + return final_score diff --git a/himotoki/grammar/splits.py b/himotoki/grammar/splits.py new file mode 100644 index 0000000..4105a6a --- /dev/null +++ b/himotoki/grammar/splits.py @@ -0,0 +1,713 @@ +""" +Split definitions module for himotoki. +Ports ichiran's dict-split.lisp split functionality. + +Splits allow compound words to be scored as the sum of their parts. +For example: 一人で → 一人 + で +This helps correctly segment expressions that appear as single entries +in the dictionary but are better understood as compound phrases. + +Types of splits: +1. Regular splits (*split_map*): Split during scoring +2. Segment splits (*segsplit_map*): Expand one segment into several during path finding +""" + +from dataclasses import dataclass, field +from typing import Optional, List, Dict, Tuple, Union, Any, Callable, Set +from functools import lru_cache + +from sqlalchemy import select, and_, or_ +from sqlalchemy.orm import Session + +from himotoki.db.models import ( + Entry, KanjiText, KanaText, Sense, SenseProp, + Conjugation, ConjProp, +) +from himotoki.characters import is_kana, as_hiragana + + +# ============================================================================ +# Split Result Types +# ============================================================================ + +@dataclass +class SplitPart: + """A part of a split compound word.""" + reading: Any # WordMatch or similar + text: str + + def __repr__(self): + return f"" + + +@dataclass +class SplitResult: + """ + Result of splitting a word. + + Supports three scoring modes via modifiers: + - Standard (no modifier): Sum of part scores + score_bonus + - ':score': Direct score addition (score_bonus added directly) + - ':pscore': Proportional score modification (score_bonus modifies prop_score) + """ + parts: List[SplitPart] + score_bonus: int + modifiers: Set[str] = field(default_factory=set) # ':score', ':pscore', or empty for standard + + def __repr__(self): + texts = [p.text for p in self.parts] + mod_str = f", modifiers={self.modifiers}" if self.modifiers else "" + return f"" + + +# ============================================================================ +# Split Maps +# ============================================================================ + +# Main split map: seq -> split function +_split_map: Dict[int, Callable] = {} + +# Segment split map: seq -> split function (for expanding segments) +_segsplit_map: Dict[int, Callable] = {} + + +def register_split(seq: int, func: Callable, is_segsplit: bool = False): + """Register a split function for a sequence.""" + target_map = _segsplit_map if is_segsplit else _split_map + target_map[seq] = func + + +# ============================================================================ +# Split Definition Macros +# ============================================================================ + +def def_simple_split( + seq: int, + score: int, + parts: List[Tuple], + is_segsplit: bool = False, +): + """ + Define a simple split for a word. + + Args: + seq: The entry sequence number + score: Score bonus for this split + parts: List of (seq, length) or (seq, length, conjugated) tuples + is_segsplit: If True, register as segment split instead + + Each part tuple: + - seq: Sequence number(s) of the part + - length: Length of this part (None for rest of string, int for fixed) + - conjugated: If True, look up conjugated form + """ + def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + offset = 0 + result_parts = [] + + for part_def in parts: + # Parse part definition + if len(part_def) == 2: + part_seqs, part_length = part_def + conjugated = False + else: + part_seqs, part_length, conjugated = part_def + + # Ensure seqs is a list + if not isinstance(part_seqs, (list, tuple)): + part_seqs = [part_seqs] + + # Calculate actual length + if part_length is None: + # Rest of string + part_text = text[offset:] + elif callable(part_length): + actual_len = part_length(text, offset) + if actual_len is None: + return None + part_text = text[offset:offset + actual_len] + else: + part_text = text[offset:offset + part_length] + + if not part_text: + return None + + # Find the word + if conjugated: + words = find_word_conj_of(session, part_text, *part_seqs) + else: + words = find_word_seq(session, part_text, *part_seqs) + + if not words: + return None + + result_parts.append(SplitPart(reading=words[0], text=part_text)) + + if part_length is not None: + if callable(part_length): + offset += part_length(text, offset) + else: + offset += part_length + + return SplitResult(parts=result_parts, score_bonus=score) + + register_split(seq, split_fn, is_segsplit) + return split_fn + + +def def_de_split(seq: int, seq_a: int, score: int = 20): + """ + Define a で split: word that ends with で. + Common pattern: X + で where X is the main meaning. + """ + def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + if not text.endswith('で'): + return None + + main_text = text[:-1] + de_text = 'で' + + # Find main part + main_words = find_word_seq(session, main_text, seq_a) + if not main_words: + return None + + # Find で particle + de_words = find_word_seq(session, de_text, 2028980) + if not de_words: + return None + + return SplitResult( + parts=[ + SplitPart(reading=main_words[0], text=main_text), + SplitPart(reading=de_words[0], text=de_text), + ], + score_bonus=score + ) + + register_split(seq, split_fn) + return split_fn + + +def def_toori_split(seq: int, seq_a: int, score: int = 50, seq_b: int = 1432930): + """ + Define a 通り split: word + 通り. + """ + def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + word_type = getattr(reading, 'word_type', 'kana') + + # Only for kanji readings + if word_type != 'kanji': + return None + + if len(text) < 3: + return None + + main_text = text[:-2] + toori_text = text[-2:] + + # Find main part + main_words = find_word_seq(session, main_text, seq_a) + if not main_words: + return None + + # Find 通り + toori_words = find_word_seq(session, toori_text, seq_b) + if not toori_words: + return None + + return SplitResult( + parts=[ + SplitPart(reading=main_words[0], text=main_text), + SplitPart(reading=toori_words[0], text=toori_text), + ], + score_bonus=score + ) + + register_split(seq, split_fn) + return split_fn + + +# ============================================================================ +# Helper Functions +# ============================================================================ + +def find_word_seq(session: Session, text: str, *seqs: int) -> List[Any]: + """Find word by text that matches specific sequences.""" + from himotoki.lookup import WordMatch + + table = KanaText if is_kana(text) else KanjiText + + results = session.execute( + select(table).where(and_( + table.text == text, + table.seq.in_(seqs) + )) + ).scalars().all() + + return [WordMatch(reading=r) for r in results] + + +def find_word_conj_of(session: Session, text: str, *seqs: int) -> List[Any]: + """Find word that is conjugation of specific sequences.""" + from himotoki.lookup import WordMatch + + table = KanaText if is_kana(text) else KanjiText + + # Direct match + direct = session.execute( + select(table).where(and_( + table.text == text, + table.seq.in_(seqs) + )) + ).scalars().all() + + # Conjugation match + indirect = session.execute( + select(table) + .join(Conjugation, table.seq == Conjugation.seq) + .where(and_( + table.text == text, + Conjugation.from_seq.in_(seqs) + )) + ).scalars().all() + + return [WordMatch(reading=r) for r in direct + indirect] + + +# ============================================================================ +# Split Lookup Functions +# ============================================================================ + +def get_split( + session: Session, + reading: Any, + conj_of: Optional[List[int]] = None, +) -> Optional[SplitResult]: + """ + Get split for a reading if one exists. + + Args: + session: Database session + reading: The reading to split + conj_of: List of sequence numbers this is conjugated from + + Returns: + SplitResult if split found, None otherwise + """ + seq = getattr(reading, 'seq', None) + if seq is None: + return None + + # Try direct match + split_fn = _split_map.get(seq) + if split_fn: + result = split_fn(session, reading) + if result and all(p.reading for p in result.parts): + return result + + # Try conjugation sources + if conj_of: + for cseq in conj_of: + split_fn = _split_map.get(cseq) + if split_fn: + result = split_fn(session, reading) + if result and all(p.reading for p in result.parts): + return result + + return None + + +def get_segsplit(session: Session, segment: Any) -> Optional[Any]: + """ + Get segment split for expanding a segment. + + Args: + session: Database session + segment: The segment to expand + + Returns: + New segment with expanded compound, or None + """ + from himotoki.lookup import Segment + + word = segment.word + seq = getattr(word, 'seq', None) + if seq is None: + return None + + # Get sequence set from info + seq_set = segment.info.get('seq_set', []) + conj_of = seq_set[1:] if len(seq_set) > 1 else [] + + # Try direct match + split_fn = _segsplit_map.get(seq) + if split_fn: + result = split_fn(session, word) + if result and all(p.reading for p in result.parts): + return _create_split_segment(segment, result) + + # Try conjugation sources + for cseq in conj_of: + split_fn = _segsplit_map.get(cseq) + if split_fn: + result = split_fn(session, word) + if result and all(p.reading for p in result.parts): + return _create_split_segment(segment, result) + + return None + + +def _create_split_segment(original_segment: Any, split_result: SplitResult) -> Any: + """Create a new segment from a split result.""" + from himotoki.lookup import Segment, CompoundWord + + # Create compound word from parts + parts = [p.reading for p in split_result.parts] + text = ''.join(p.text for p in split_result.parts) + + # Build kana with spaces + kana_parts = [] + for p in split_result.parts: + r = p.reading + if hasattr(r, 'reading'): + r = r.reading + kana_parts.append(getattr(r, 'text', p.text)) + + compound = CompoundWord( + primary=parts[0], + words=parts, + text=text, + kana=' '.join(kana_parts), + score_mod=split_result.score_bonus, + ) + + # Create new segment + new_seg = Segment( + start=original_segment.start, + end=original_segment.end, + word=compound, + text=text, + score=original_segment.score + split_result.score_bonus, + info=dict(original_segment.info), + ) + + return new_seg + + +# ============================================================================ +# Split Definitions - Port from dict-split.lisp +# ============================================================================ + +def init_splits(): + """Initialize all split definitions.""" + global _split_map, _segsplit_map + + if _split_map: + return # Already initialized + + # ====================== + # -de expressions + # ====================== + def_de_split(1163700, 1576150) # 一人で + def_de_split(1611020, 1577100) # 何で + def_de_split(1004800, 1628530) # これで + def_de_split(2810720, 1004820) # 此れまでで + def_de_split(1006840, 1006880) # その上で + def_de_split(1530610, 1530600) # 無断で + def_de_split(1245390, 1245290) # 空で + def_de_split(2719270, 1445430) # 土足で + def_de_split(1189420, 2416780) # 何用で + def_de_split(1272220, 1592990) # 交代で + def_de_split(1311360, 1311350) # 私費で + def_de_split(1368500, 1368490) # 人前で + def_de_split(1395670, 1395660) # 全体で + def_de_split(1417790, 1417780) # 単独で + def_de_split(1454270, 1454260) # 道理で + def_de_split(1479100, 1679020) # 半眼で + def_de_split(1510140, 1680900) # 別封で + def_de_split(1518550, 1529560) # 無しで + def_de_split(1531420, 1531410) # 名義で + def_de_split(1597400, 1585205) # 力尽くで + def_de_split(1679990, 2582460) # 抜き足で + def_de_split(1682060, 2085340) # 金ずくで + def_de_split(1736650, 1611710) # 水入らずで + def_de_split(1865020, 1590150) # 陰で + def_de_split(1878880, 2423450) # 差しで + def_de_split(2126220, 1802920) # 捩じり鉢巻きで + def_de_split(2136520, 2005870) # もう少しで + def_de_split(2513590, 2513650) # 詰め開きで + def_de_split(2771850, 2563780) # 気にしないで + def_de_split(2810800, 1587590) # 今までで + def_de_split(1343110, 1343100) # ところで + def_de_split(1270210, 1001640) # お陰で + + # ====================== + # -通り expressions + # ====================== + def_toori_split(1260990, 1260670) # 元通り + def_toori_split(1414570, 2082450) # 大通り + def_toori_split(1424950, 1620400) # 中通り (ちゅう通り) + def_toori_split(1424960, 1423310) # 中通り (なか通り) + def_toori_split(1820790, 1250090) # 型通り + def_toori_split(1489800, 1489340) # 表通り + def_toori_split(1523010, 1522150) # 本通り + def_toori_split(1808080, 1604890) # 目通り + def_toori_split(1368820, 1580640) # 人通り + def_toori_split(1550490, 1550190) # 裏通り + def_toori_split(1619440, 2069220) # 素通り + def_toori_split(1164910, 2821500, seq_b=1432920) # 一通り + def_toori_split(1462720, 1461140, seq_b=1432920) # 二通り + + # ====================== + # ど- prefix splits + # ====================== + _register_do_split(2142710, 1185200) # ど下手 + _register_do_split(2803190, 1595630) # どすけべ + _register_do_split(2142680, 1290210) # ど根性 + _register_do_split(2523480, 1442750) # ど田舎 + + # ====================== + # し- splits (する conjugations) + # ====================== + _register_shi_split(1005700, 1156990) # し易い + _register_shi_split(1005830, 1370760) # し吹く + _register_shi_split(1157200, 2772730) # し難い + _register_shi_split(1157220, 1195970) # し過ぎる + _register_shi_split(1157230, 1284430) # し合う + _register_shi_split(1157280, 1370090) # し尽す + _register_shi_split(1157310, 1405800) # し続ける + _register_shi_split(1304890, 1256520) # し兼ねる + _register_shi_split(1304960, 1307550) # し始める + _register_shi_split(1305110, 1338180) # し出す + _register_shi_split(1305280, 1599390) # し直す + _register_shi_split(1305290, 1212670) # し慣れる + _register_shi_split(1594300, 1596510) # し損なう + _register_shi_split(1594310, 1406680) # し損じる + _register_shi_split(1594460, 1372620) # し遂げる + _register_shi_split(1594580, 1277100) # し向ける + _register_shi_split(2518250, 1332760) # し終える + _register_shi_split(1157240, 1600260) # し残す + _register_shi_split(1304820, 1207610) # し掛ける + _register_shi_split(2858937, 1406690) # し損ねる + + # ====================== + # Complex splits + # ====================== + + # なくなる (無くなる) + def split_nakunaru(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + naku = find_word_conj_of(session, text[:2], 1529520) + naru = find_word_conj_of(session, text[2:], 1375610) + if naku and naru: + return SplitResult( + parts=[ + SplitPart(reading=naku[0], text=text[:2]), + SplitPart(reading=naru[0], text=text[2:]), + ], + score_bonus=30 + ) + return None + register_split(1529550, split_nakunaru) + + # という + def split_toiu(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + to = find_word_seq(session, 'と', 1008490) + iu = find_word_conj_of(session, text[1:], 1587040) + if to and iu: + return SplitResult( + parts=[ + SplitPart(reading=to[0], text='と'), + SplitPart(reading=iu[0], text=text[1:]), + ], + score_bonus=20 + ) + return None + register_split(1922760, split_toiu) + + # じゃない + def split_janai(session: Session, reading: Any) -> Optional[SplitResult]: + ja = find_word_seq(session, 'じゃ', 2089020) + nai = find_word_conj_of(session, 'ない', 1529520) + if ja and nai: + return SplitResult( + parts=[ + SplitPart(reading=ja[0], text='じゃ'), + SplitPart(reading=nai[0], text='ない'), + ], + score_bonus=10 + ) + return None + register_split(2755350, split_janai) + + # なら + def split_nara(session: Session, reading: Any) -> Optional[SplitResult]: + nara = find_word_conj_of(session, 'なら', 2089020) + if nara: + return SplitResult( + parts=[SplitPart(reading=nara[0], text='なら')], + score_bonus=1 + ) + return None + register_split(1009470, split_nara) + + # 気がつく + def split_kigatsuku(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + ki = find_word_seq(session, '気', 1221520) + ga = find_word_seq(session, 'が', 2028930) + tsuku = find_word_conj_of(session, text[2:], 1495740) + if ki and ga and tsuku: + return SplitResult( + parts=[ + SplitPart(reading=ki[0], text='気'), + SplitPart(reading=ga[0], text='が'), + SplitPart(reading=tsuku[0], text=text[2:]), + ], + score_bonus=100 + ) + return None + register_split(1591050, split_kigatsuku) + + # 気のせい + def split_kinosei(session: Session, reading: Any) -> Optional[SplitResult]: + ki = find_word_seq(session, '気', 1221520) + no = find_word_seq(session, 'の', 1469800) + sei = find_word_seq(session, 'せい', 1610040) + if ki and no and sei: + return SplitResult( + parts=[ + SplitPart(reading=ki[0], text='気'), + SplitPart(reading=no[0], text='の'), + SplitPart(reading=sei[0], text='せい'), + ], + score_bonus=100 + ) + return None + register_split(1221750, split_kinosei) + + # ====================== + # Segment splits + # ====================== + + # ところが + def segsplit_tokoroga(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + tokoro = find_word_seq(session, text[:-1], 1343100) + ga = find_word_seq(session, 'が', 2028930) + if tokoro and ga: + return SplitResult( + parts=[ + SplitPart(reading=tokoro[0], text=text[:-1]), + SplitPart(reading=ga[0], text='が'), + ], + score_bonus=-10 + ) + return None + register_split(1008570, segsplit_tokoroga, is_segsplit=True) + + # ところで + def segsplit_tokorode(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + tokoro = find_word_seq(session, text[:-1], 1343100) + de = find_word_seq(session, 'で', 2028980) + if tokoro and de: + return SplitResult( + parts=[ + SplitPart(reading=tokoro[0], text=text[:-1]), + SplitPart(reading=de[0], text='で'), + ], + score_bonus=-10 + ) + return None + register_split(1343110, segsplit_tokorode, is_segsplit=True) + + # とは + def segsplit_toha(session: Session, reading: Any) -> Optional[SplitResult]: + to = find_word_seq(session, 'と', 1008490) + ha = find_word_seq(session, 'は', 2028920) + if to and ha: + return SplitResult( + parts=[ + SplitPart(reading=to[0], text='と'), + SplitPart(reading=ha[0], text='は'), + ], + score_bonus=-5 + ) + return None + register_split(2028950, segsplit_toha, is_segsplit=True) + + # では + def segsplit_deha(session: Session, reading: Any) -> Optional[SplitResult]: + de = find_word_seq(session, 'で', 2028980) + ha = find_word_seq(session, 'は', 2028920) + if de and ha: + return SplitResult( + parts=[ + SplitPart(reading=de[0], text='で'), + SplitPart(reading=ha[0], text='は'), + ], + score_bonus=-5 + ) + return None + register_split(1008450, segsplit_deha, is_segsplit=True) + + # だから + def segsplit_dakara(session: Session, reading: Any) -> Optional[SplitResult]: + da = find_word_seq(session, 'だ', 2089020) + kara = find_word_seq(session, 'から', 1002980) + if da and kara: + return SplitResult( + parts=[ + SplitPart(reading=da[0], text='だ'), + SplitPart(reading=kara[0], text='から'), + ], + score_bonus=-5 + ) + return None + register_split(1007310, segsplit_dakara, is_segsplit=True) + + +def _register_do_split(seq: int, seq_b: int, score: int = 30): + """Register a ど- prefix split.""" + def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + do = find_word_seq(session, 'ど', 2252690) + rest = find_word_seq(session, text[1:], seq_b) + if do and rest: + return SplitResult( + parts=[ + SplitPart(reading=do[0], text='ど'), + SplitPart(reading=rest[0], text=text[1:]), + ], + score_bonus=score + ) + return None + register_split(seq, split_fn) + + +def _register_shi_split(seq: int, seq_b: int, score: int = 30): + """Register a し- (する conjugation) split.""" + def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: + text = getattr(reading, 'text', str(reading)) + shi = find_word_conj_of(session, 'し', 1157170) + rest = find_word_conj_of(session, text[1:], seq_b) + if shi and rest: + return SplitResult( + parts=[ + SplitPart(reading=shi[0], text='し'), + SplitPart(reading=rest[0], text=text[1:]), + ], + score_bonus=score + ) + return None + register_split(seq, split_fn) + + +# Initialize on module load +init_splits() diff --git a/himotoki/grammar/suffix_handlers.py b/himotoki/grammar/suffix_handlers.py new file mode 100644 index 0000000..62498ed --- /dev/null +++ b/himotoki/grammar/suffix_handlers.py @@ -0,0 +1,619 @@ +""" +Suffix handler functions for compound word matching. +""" + +from typing import Optional, List, Dict, Tuple, Union, Any, Callable + +from sqlalchemy.orm import Session + +from himotoki.db.models import KanaText +from himotoki.constants import ( + SEQ_SURU, + BLOCKED_NAI_SEQS, BLOCKED_NAI_X_SEQS, +) + +from himotoki.grammar.suffixes import ( + find_word_with_conj_type, + find_word_with_pos, + find_word_with_neg_prop, + find_word_suffix, +) + +def _handler_tai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle たい suffix - want to...""" + if root == 'い': + return [] + return find_word_with_conj_type(session, root, 13) # Continuative + + +def _handler_ren(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle generic ren'youkei (continuative) suffix.""" + return find_word_with_conj_type(session, root, 13) + + +def _handler_teren(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle te-form + ren'youkei suffix (e.g., やがる - disdain).""" + return find_word_with_conj_type(session, root, 13) # Continuative + + +def _handler_neg(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle negative stem suffix.""" + from himotoki.lookup import CONJ_NEGATIVE_STEM + return find_word_with_conj_type(session, root, 13, CONJ_NEGATIVE_STEM) + + +def _handler_chau(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """ + Handle ちゃう suffix - contracted てしまう (completion). + + Ports ichiran's suffix-chau from dict-grammar.lisp. + The suffix starts with ち or じ (contracted from て or で). + We reconstruct the te-form by: + 1. Looking at first char of suffix to determine て or で + 2. Concatenating root + て/で to form the te-form + 3. Looking up that te-form in the database + + For example: サボっちゃった + - root = "サボっ" + - suffix = "ちゃった" (first char is ち → て) + - lookup "サボっ" + "て" = "サボって" as te-form + """ + if not suffix: + return [] + + # Map first char of suffix to te/de + first_char = suffix[0] + if first_char == 'ち': + te = 'て' + elif first_char == 'じ': + te = 'で' + else: + return [] + + # Look up root + te as te-form conjugation + te_form = root + te + return find_word_with_conj_type(session, te_form, 3) + + +def _handler_to_contracted(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """ + Handle とく suffix - contracted ておく (doing in advance). + + Ports ichiran's suffix-to from dict-grammar.lisp. + The suffix starts with と or ど (contracted from て or で). + We reconstruct the te-form similarly to chau. + + For example: 置いとく + - root = "置い" + - suffix = "とく" (first char is と → て) + - lookup "置い" + "て" = "置いて" as te-form + """ + if not suffix: + return [] + + # Map first char of suffix to te/de + first_char = suffix[0] + if first_char == 'と': + te = 'て' + elif first_char == 'ど': + te = 'で' + else: + return [] + + # Look up root + te as te-form conjugation + te_form = root + te + return find_word_with_conj_type(session, te_form, 3) + + +def _handler_te(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle て form suffix.""" + if root == 'で': + return [] + if not root.endswith('て') and not root.endswith('で'): + return [] + return find_word_with_conj_type(session, root, 3) # Te-form + + +def _handler_teiru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle ている suffix.""" + if root == 'いて': + return [] + if not root.endswith('て') and not root.endswith('で'): + return [] + # First try direct database lookup for te-form + results = find_word_with_conj_type(session, root, 3) + if results: + return results + # If not found, try to find root as a compound via suffix matching + # This enables nested compounds like 勉強し続けている (勉強し + 続けて + いる) + compound_results = find_word_suffix(session, root) + return compound_results + + +def _handler_suru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle する suffix - make verb from noun.""" + return find_word_with_pos(session, root, 'vs') + + +def _handler_sou(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle そう suffix - looks like. + + そう attaches to: + 1. Verb continuative form (ren'youkei): 食べそう, 降りそう + 2. Adjective stem (without い): 美しそう, 高そう + 3. Na-adjective root: 静かそう, 元気そう + 4. Negative なさ form: 情けなさそう + """ + from himotoki.lookup import CONJ_ADJECTIVE_STEM, CONJ_ADVERBIAL + if root in ('な', 'よ', 'よさ', 'に', 'き'): + return [] + + # Check for なさ ending (negative adjective) + if root.endswith('なさ'): + root_patched = root[:-1] + 'い' + return find_word_with_neg_prop(session, root_patched) + + results = find_word_with_conj_type(session, root, 13, CONJ_ADJECTIVE_STEM, CONJ_ADVERBIAL) + # Filter out なぜる (seq 10195060 for conjugated form) which incorrectly matches なぜ + そう + # The word なぜ is the interrogative "why", not the verb "to stroke" + results = [r for r in results if getattr(r, 'seq', None) != 10195060] + + # Also check for na-adjectives (静かそう, 元気そう) + if not results: + results = find_word_with_pos(session, root, 'adj-na') + + return results + + +def _handler_sugiru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle すぎる suffix - too much. + + すぎる attaches to: + 1. Verb continuative form (ren'youkei): 食べすぎる, 飲みすぎる + 2. Adjective stem (without い): 高すぎる, 美しすぎる + 3. Na-adjective root: 静かすぎる, 元気すぎる + 4. Negative なさ form: 情けなさすぎる + """ + if root == 'い': + return [] + + # Check for なさ ending + if root.endswith('なさ') or root.endswith('無さ'): + root_patched = root[:-1] + 'い' + return find_word_with_neg_prop(session, root_patched) + + results = [] + + # Try verb continuative form (ren'youkei, conj_type=13) + results.extend(find_word_with_conj_type(session, root, 13)) + + # Try adjective stem (add い and look for adj-i) + root_i = root + 'い' + results.extend(find_word_with_pos(session, root_i, 'adj-i')) + + # Try na-adjective root: 静かすぎる, 元気すぎる + results.extend(find_word_with_pos(session, root, 'adj-na')) + + return results + + +def _handler_sa(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle さ suffix - -ness.""" + from himotoki.lookup import CONJ_ADJECTIVE_STEM + result = find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) + result.extend(find_word_with_pos(session, root, 'adj-na')) + return result + + +def _handler_rou(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle だろう suffix - probably/conjecture. + + だろう attaches to: + 1. Verb dictionary form: 食べるだろう, 行くだろう + 2. Verb past form: 食べただろう + 3. Adjective dictionary form: 高いだろう + 4. Na-adjective root: 静かだろう + 5. Negative form: 食べないだろう + """ + results = [] + + # Try dictionary form (direct word lookup) + results.extend(find_word_with_pos(session, root, 'v1')) + results.extend(find_word_with_pos(session, root, 'v5')) + results.extend(find_word_with_pos(session, root, 'adj-i')) + results.extend(find_word_with_pos(session, root, 'adj-na')) + + # Try past form (conj_type=2) + results.extend(find_word_with_conj_type(session, root, 2)) + + # Try negative form + if root.endswith('ない'): + results.extend(find_word_with_neg_prop(session, root)) + + return results + + +def _handler_adv(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle adverbial suffix.""" + from himotoki.lookup import CONJ_ADVERBIAL + return find_word_with_conj_type(session, root, CONJ_ADVERBIAL) + + +def _handler_kudasai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle ください suffix - please do.""" + if not root.endswith('て') and not root.endswith('で'): + return [] + return find_word_with_conj_type(session, root, 3) + + +def _handler_teii(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle ていい suffix - ok if.""" + if not root.endswith('て') and not root.endswith('で'): + return [] + return find_word_with_conj_type(session, root, 3) + + +def _handler_garu(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle がる suffix - feel/show signs of. + + がる attaches to adjective stems: + 1. i-adjective stem: 欲しがる (from 欲しい) + 2. tai-compound stem: 食べたがる (from 食べたい = 食べる + たい) + 3. sou-compound stem: よさそうがる + """ + from himotoki.lookup import CONJ_ADJECTIVE_STEM + if root in ('な', 'い', 'よ'): + return [] + + result = find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) + + # If no direct adjective match, check for tai compound chain + # e.g., 食べたがる: root='食べた' → root+'い' = '食べたい' → tai compound + if not result and root.endswith('た'): + tai_form = root + 'い' # Reconstruct potential たい form + tai_results = find_word_suffix(session, tai_form) + result.extend(tai_results) + + # Also check for そ ending (そう + がる) + if root.endswith('そ'): + root_patched = root[:-1] + 'う' + result.extend(find_word_with_suffix(session, root_patched, 'sou')) + + return result + + +def _handler_nade(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle na-adjective て-form (で). + + で is the conjunctive/te-form for na-adjectives via copula だ: + 静かで (being quiet; quietly and...), 元気で (being healthy), etc. + """ + return find_word_with_pos(session, root, 'adj-na') + + +def _handler_ra(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle ら suffix - plural.""" + if root.endswith('ら'): + return [] + return find_word_with_pos(session, root, 'pn') + + +def _handler_ppoi(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle っぽい suffix - -ish / tends to. + + っぽい attaches to: + 1. Verb continuative stem: 忘れ + っぽい, 飽き + っぽい + 2. Nouns: 子供 + っぽい, 大人 + っぽい, 水 + っぽい + 3. Na-adjective root: 静か + っぽい + + Note: the っ is part of the suffix, so root already has it stripped. + """ + result = find_word_with_conj_type(session, root, 13) # Verb continuative + result.extend(find_word_with_pos(session, root, 'n')) + result.extend(find_word_with_pos(session, root, 'adj-na')) + return result + + +def _handler_mi(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle み suffix - adjective nominalization (-ness). + + み converts adjectives to nouns: + - i-adj: 深い → 深み (depth), 甘い → 甘み (sweetness) + - na-adj: 静か → 静かみ (quietness) + + Root will be the adjective stem (without い for i-adj, or bare form for na-adj). + """ + from himotoki.lookup import CONJ_ADJECTIVE_STEM + result = find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) + result.extend(find_word_with_pos(session, root, 'adj-na')) + return result + + +def _handler_tachi(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle たち suffix - plural for people/animals. + + たち attaches to nouns and pronouns: + 学生たち, 子供たち, 私たち, 猫たち + """ + result = find_word_with_pos(session, root, 'n') + result.extend(find_word_with_pos(session, root, 'pn')) + return result + + +def _handler_rashii(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle らしい suffix - seems like. + + らしい attaches to: + 1. Verb/adj conjugated forms (conj_type 2) + 2. ら-ending forms (conj_type 11) + 3. Nouns: 男らしい, 春らしい, 学生らしい + 4. Na-adjective stems: 静からしい + """ + result = find_word_with_conj_type(session, root, 2) + result.extend(find_word_with_conj_type(session, root + 'ら', 11)) + # Also try noun lookup for noun+rashii patterns + result.extend(find_word_with_pos(session, root, 'n')) + result.extend(find_word_with_pos(session, root, 'adj-na')) + return result + + +def _handler_desu(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle です suffix - formal copula.""" + # Negative copula forms (e.g., 〜ないです) + if root.endswith('ない') or root.endswith('なかった'): + return find_word_with_neg_prop(session, root) + + # na-adjective + copula (e.g., 大丈夫です, 静かでした) + if len(root) < 2: + return [] + return find_word_with_pos(session, root, 'adj-na') + + +def _handler_desho(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle でしょう suffix - polite conjecture. + + でしょう attaches to: + 1. Verb dictionary form: 食べるでしょう + 2. Verb past form: 食べたでしょう + 3. Adjective dictionary form: 高いでしょう + 4. Na-adjective root: 静かでしょう + 5. Negative form: 食べないでしょう + """ + results = [] + + # Try dictionary form (direct word lookup) + results.extend(find_word_with_pos(session, root, 'v1')) + results.extend(find_word_with_pos(session, root, 'v5')) + results.extend(find_word_with_pos(session, root, 'adj-i')) + results.extend(find_word_with_pos(session, root, 'adj-na')) + + # Try past form (conj_type=2) + results.extend(find_word_with_conj_type(session, root, 2)) + + # Try negative form + if root.endswith('ない'): + results.extend(find_word_with_neg_prop(session, root)) + + return results + + +def _handler_tosuru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle とする suffix - try to.""" + return find_word_with_conj_type(session, root, 9) # Volitional + + +def _handler_kurai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle くらい suffix - about/approximately. + + くらい attaches to: + 1. Verb dictionary form: 食べるくらい + 2. Verb past form: 食べたくらい + 3. Verb continuative form: 食べくらい (literary) + 4. Noun/pronoun: それくらい + """ + results = [] + # Try past form + results.extend(find_word_with_conj_type(session, root, 2)) + # Try continuative form + results.extend(find_word_with_conj_type(session, root, 13)) + # Try dictionary form (verbs, adjectives) + results.extend(find_word_with_pos(session, root, 'v1')) + results.extend(find_word_with_pos(session, root, 'v5')) + results.extend(find_word_with_pos(session, root, 'adj-i')) + results.extend(find_word_with_pos(session, root, 'adj-na')) + return results + + +def _handler_iadj(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle i-adjective suffix like げ, め.""" + from himotoki.lookup import CONJ_ADJECTIVE_STEM + return find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) + + +# Abbreviation handlers + +def _find_word_with_neg_prop_filtered( + session: Session, + word: str, + blocked_seqs: set, + allow_root: bool = False, +) -> List[Any]: + """ + Find words with negative conjugation property, excluding certain root seqs. + + Ports ichiran's filter logic from abbr-nee and abbr-n in dict-grammar.lisp: + (and (not (find (conj-data-from cdata) '(1577980 1547720))) + (conj-neg (conj-data-prop cdata))) + + Args: + session: Database session + word: Word text to search for + blocked_seqs: Set of from_seq values to exclude + allow_root: If True, also return root forms + + Returns: + List of word matches with negative conjugation, excluding blocked seqs + """ + from himotoki.lookup import find_word_with_conj_prop + + def filter_fn(cdata): + # Must be negative form + if not (cdata.prop and hasattr(cdata.prop, 'neg') and cdata.prop.neg): + return False + # Exclude blocked root seqs (居ない, 来ない create problems) + if cdata.from_seq in blocked_seqs: + return False + return True + + return find_word_with_conj_prop(session, word, filter_fn, allow_root=allow_root) + + +def _handler_abbr_nai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """ + Handle ない abbreviation (ねえ, ねぇ, ねー etc.). + + Ports ichiran's abbr-nee from dict-grammar.lisp. + Blocks いる (SEQ_IRU) and 来る (SEQ_KURU) conjugations to avoid false matches. + Allows root forms (:allow-root t in ichiran). + """ + return _find_word_with_neg_prop_filtered( + session, root + 'ない', BLOCKED_NAI_SEQS, allow_root=True + ) + + +def _handler_abbr_nai_n(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """ + Handle ん contraction (nai-n suffix). + + Ports ichiran's abbr-n from dict-grammar.lisp. + Blocks いる (SEQ_IRU) and 来る (SEQ_KURU) conjugations to avoid false matches. + Does NOT allow root forms (differs from abbr-nee). + + Example: 考えてん should NOT match 考えていないん (negative of いる). + Instead, it should match 考えて + ん separately. + """ + return _find_word_with_neg_prop_filtered( + session, root + 'ない', BLOCKED_NAI_SEQS, allow_root=False + ) + + +def _handler_abbr_nx(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """ + Handle ず/ざる/ぬ abbreviation (nai-x suffix). + + Ports ichiran's abbr-nx from dict-grammar.lisp. + Blocks する (SEQ_SURU) and 富む (SEQ_TOMU) conjugations. + Special case: せ -> しない (for する). + """ + if root == 'せ': + from himotoki.grammar.splits import find_word_conj_of + return find_word_conj_of(session, 'しない', SEQ_SURU) + + from himotoki.lookup import find_word_with_conj_prop + + def filter_fn(cdata): + # Must be negative form + if not (cdata.prop and hasattr(cdata.prop, 'neg') and cdata.prop.neg): + return False + # Exclude blocked words (する, 富む) + if cdata.from_seq in BLOCKED_NAI_X_SEQS: + return False + return True + + return find_word_with_conj_prop(session, root + 'ない', filter_fn) + + +def _handler_abbr_nakereba(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle なきゃ/なくちゃ abbreviation.""" + from himotoki.lookup import find_word_full + return find_word_full(session, root + 'なければ') + + +def _handler_abbr_shimasho(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle ましょ abbreviation (of ましょう - polite volitional).""" + from himotoki.lookup import find_word_full + return find_word_full(session, root + 'ましょう') + + +def _handler_abbr_dewanai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle じゃない abbreviation.""" + from himotoki.lookup import find_word_full + return find_word_full(session, root + 'ではない') + + +def _handler_abbr_eba(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle conditional abbreviations (りゃ, きゃ, etc.).""" + from himotoki.lookup import find_word_full + # Map abbreviation to full conditional form + suffix_map = { + 'ちゃ': 'てば', + 'りゃ': 'れば', + 'きゃ': 'けば', + 'ぎゃ': 'げば', + 'にゃ': 'ねば', + 'びゃ': 'べば', + 'みゃ': 'めば', + 'しゃ': 'せば', + } + full_suffix = suffix_map.get(suffix) + if full_suffix: + return find_word_full(session, root + full_suffix) + return [] + + +def _handler_abbr_ii(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: + """Handle ええ abbreviation for いい.""" + from himotoki.lookup import find_word_full + return find_word_full(session, root + 'いい') + + +# Mapping of suffix keywords to handlers +SUFFIX_HANDLERS: Dict[str, Callable] = { + 'tai': _handler_tai, + 'ren': _handler_ren, + 'ren+': _handler_ren, + 'ren-': _handler_ren, + 'neg': _handler_neg, + 'te': _handler_te, + 'teiru': _handler_teiru, + 'teiru+': _handler_teiru, + 'te+space': _handler_te, + 'suru': _handler_suru, + 'sou': _handler_sou, + 'sou+': _handler_sou, + 'sugiru': _handler_sugiru, + 'sa': _handler_sa, + 'adv': _handler_adv, + 'kudasai': _handler_kudasai, + 'teii': _handler_teii, + 'garu': _handler_garu, + 'teren': _handler_teren, + 'ra': _handler_ra, + 'rashii': _handler_rashii, + 'desu': _handler_desu, + 'tosuru': _handler_tosuru, + 'kurai': _handler_kurai, + 'iadj': _handler_iadj, + 'mi': _handler_mi, + 'nade': _handler_nade, + 'ppoi': _handler_ppoi, + 'tachi': _handler_tachi, + 'chau': _handler_chau, # Contracted てしまう with te-reconstruction + 'to': _handler_to_contracted, # Contracted ておく with te-reconstruction + # Abbreviations - each has distinct behavior matching ichiran + 'nai': _handler_abbr_nai, # ねえ, ねぇ, ねー - allows root forms + 'nai-x': _handler_abbr_nx, # ず, ざる, ぬ - blocks する + 'nai-n': _handler_abbr_nai_n, # ん contraction - blocks いる/来る, no root + 'nakereba': _handler_abbr_nakereba, + 'shimashou': _handler_abbr_shimasho, + 'dewanai': _handler_abbr_dewanai, + 'teba': _handler_abbr_eba, + 'reba': _handler_abbr_eba, + 'keba': _handler_abbr_eba, + 'geba': _handler_abbr_eba, + 'neba': _handler_abbr_eba, + 'beba': _handler_abbr_eba, + 'meba': _handler_abbr_eba, + 'seba': _handler_abbr_eba, + 'ii': _handler_abbr_ii, +} diff --git a/himotoki/grammar/suffixes.py b/himotoki/grammar/suffixes.py new file mode 100644 index 0000000..b538b40 --- /dev/null +++ b/himotoki/grammar/suffixes.py @@ -0,0 +1,1157 @@ +""" +Suffix handling module for himotoki. +Ports ichiran's dict-grammar.lisp suffix functionality. + +Suffixes are grammatical endings that attach to word stems to form compound words. +Examples include: +- たい (tai): want to... +- ている (teiru): indicates continuing action +- そう (sou): looks like... +- たくない (takunai): don't want to... + +The suffix cache maps suffix strings to (keyword, kana_form) pairs. +This enables efficient lookup during segmentation. +""" + +from dataclasses import dataclass, field +from typing import Optional, List, Dict, Tuple, Union, Any, Callable, Set +from functools import lru_cache +import threading + +from sqlalchemy import select, and_, or_ +from sqlalchemy.orm import Session + +from himotoki.db.models import ( + Entry, KanjiText, KanaText, Sense, SenseProp, + Conjugation, ConjProp, ConjSourceReading, +) +from himotoki.characters import is_kana, as_hiragana +from himotoki.constants import ( + CONJ_ADVERBIAL, CONJ_ADJECTIVE_STEM, + # Particles + SEQ_O_PREFIX, SEQ_DE, SEQ_KA, SEQ_NI, SEQ_WO, SEQ_NO, SEQ_TTE, SEQ_KARA, + SEQ_WA, SEQ_MO, + # Common verbs + SEQ_IRU, SEQ_KURU, SEQ_SURU, SEQ_TOMU, SEQ_ORU, SEQ_ARU, SEQ_OKU, SEQ_IKU, + SEQ_SHIMAU, SEQ_MORAU, SEQ_ITADAKU, SEQ_KURERU, SEQ_MIRU, SEQ_AGERU, + SEQ_HOSHII, + # Honorific/humble verb forms + SEQ_ITASU, SEQ_SARERU, SEQ_SASERU, SEQ_TOKU, + # Suffix-related + SEQ_CHAU, SEQ_CHIMAU, SEQ_TAI, SEQ_NIKUI, SEQ_II, SEQ_KUDASAI, + SEQ_SOU, SEQ_SOU_NI_NAI, SEQ_MOII, SEQ_NAGARA, + SEQ_PPOI, SEQ_GATAI, SEQ_DASU, SEQ_KIRU, SEQ_KATA, SEQ_MI, + SEQ_YASUI, SEQ_MAKURU, SEQ_NAOSU, SEQ_SOKONAU, SEQ_WASURERU, + SEQ_OERU, SEQ_ZURAI, SEQ_GIMI, SEQ_PPANASHI, SEQ_TACHI, + SEQ_AU, SEQ_KOMU, SEQ_HOUDAI, SEQ_OWARU, SEQ_HAJIMERU, SEQ_TSUKERU, + SEQ_YARU, SEQ_MAIRU, SEQ_KUDASARU, SEQ_SASHIAGERU, + # Blocked seqs + BLOCKED_NAI_SEQS, BLOCKED_NAI_X_SEQS, + # Suffix descriptions (to merge with local ones) + SUFFIX_DESCRIPTION as SUFFIX_DESCRIPTION_SEQS, +) + + +# ============================================================================ +# Suffix Descriptions +# ============================================================================ + +SUFFIX_DESCRIPTION: Dict[Union[str, int], str] = { + # Verbal suffixes + 'chau': 'indicates completion / to do something by accident or regret', + 'ha': 'topic marker particle', + 'tai': 'want to... / would like to...', + 'iru': 'indicates continuing action (to be ...ing)', + 'oru': 'indicates continuing action (to be ...ing) (humble)', + 'aru': 'indicates completion / finished action', + 'kuru': 'indicates action that had been continuing up till now / came to be', + 'oku': 'to do in advance / to leave in the current state expecting a later change', + 'kureru': '(asking) to do something for one', + 'morau': '(asking) to get somebody to do something', + 'itadaku': '(asking) to get somebody to do something (polite)', + 'iku': 'is becoming / action starting now and continuing', + 'miru': 'to try doing ...', + 'ageru': 'to do something for someone', + 'hoshii': 'want someone to ...', + 'suru': 'makes a verb from a noun', + 'itasu': 'makes a verb from a noun (humble)', + 'sareru': 'makes a verb from a noun (honorific or passive)', + 'saseru': 'let/make someone/something do ...', + 'rou': 'probably / it seems that... / I guess ...', + 'ii': "it's ok if ... / is it ok if ...?", + 'mo': 'even if ...', + 'sugiru': 'to be too (much) ...', + 'nikui': 'difficult to...', + 'sa': '-ness (degree or condition of adjective)', + 'tsutsu': 'while ... / in the process of ...', + 'nagara': 'while doing ... / simultaneously ...', + 'tsutsuaru': 'to be doing ... / to be in the process of doing ...', + 'tsuzukeru': 'to continue ...', + 'uru': 'can ... / to be able to ...', + 'sou': 'looking like ... / seeming ...', + 'nai': 'negative suffix', + 'naide': 'without doing ... / don\'t', + 'ra': 'pluralizing suffix (not polite)', + 'kudasai': 'please do ...', + 'yagaru': 'indicates disdain or contempt', + 'naru': 'to become ...', + 'desu': 'formal copula', + 'desho': "it seems/perhaps/don't you think?", + 'tosuru': 'to try to .../to be about to...', + 'garu': 'to feel .../have a ... impression of someone', + 'me': 'somewhat/-ish', + 'gai': 'worth it to ...', + 'tasou': 'seem to want to... (tai+sou)', + 'nade': 'na-adj conjunctive (and/being)', + 'nakereba': 'must do / have to (contraction)', + 'shimashou': 'let\'s ... (polite volitional contraction)', + 'kurai': 'about/approximately/to the extent of', + 'ppoi': '-ish / tends to / apt to', + 'gatai': 'difficult to... / hard to... (literary)', + 'dasu': 'to burst out doing / to start suddenly', + 'kiru': 'to do completely / to finish doing', + 'kata': 'way of doing / how to ...', + 'mi': '-ness / depth of feeling (nominalization)', + 'yasui': 'easy to ... / likely to ...', + 'makuru': 'to do relentlessly / to keep doing', + 'naosu': 'to redo / to do again', + 'sokonau': 'to fail to do / to miss doing', + 'wasureru': 'to forget to do', + 'oeru': 'to finish doing', + 'zurai': 'difficult to ... / hard to ...', + 'gimi': 'tending to / -ish / -like', + 'ppanashi': 'left doing / leaving as is', + 'tachi': 'plural (people/animals)', + 'au': 'to do mutually / to do together', + 'komu': 'to do into / to do thoroughly', + 'houdai': 'as much as one likes / unlimited', + 'owaru': 'to finish doing', + 'hajimeru': 'to start doing / to begin to', + 'tsukeru': 'to be accustomed to doing', + 'yaru': 'to do for (someone, casual)', + 'mairu': 'to go/come (humble, te-form auxiliary)', + 'kudasaru': 'to kindly do for (honorific)', + 'sashiageru': 'to do for (humble, respectful)', + # Particle seqs - imported from constants and merged + **SUFFIX_DESCRIPTION_SEQS, +} + + +# ============================================================================ +# Maximum Suffix Nesting Depth +# ============================================================================ +# Limits recursion when finding nested compound words like: +# 勉強し続けている (勉強し + 続けて + いる) +# Without a limit, pathological inputs could cause infinite recursion. +MAX_SUFFIX_DEPTH = 5 + +# Thread-local storage for tracking current recursion depth +import contextvars +_current_suffix_depth: contextvars.ContextVar[int] = contextvars.ContextVar('suffix_depth', default=0) + +# ============================================================================ +# Global Suffix Cache +# ============================================================================ + +# Cache mapping suffix text to (keyword, kana_form) pairs +_suffix_cache: Dict[str, List[Tuple[str, Optional[Any]]]] = {} + +# Set of all unique 1-char suffix endings for quick filtering +# If a word doesn't end with any of these, no suffix can match +_suffix_ending_chars: Set[str] = set() + +# Mapping from seq to suffix class +_suffix_class: Dict[int, str] = {} + +# Mapping from suffix text to suffix class (for abbreviations without seq) +_suffix_text_class: Dict[str, str] = {} + +# Lock for thread-safe cache initialization +_suffix_lock = threading.Lock() + +# Flag indicating cache is initialized +_suffix_initialized = False + + +# ============================================================================ +# Suffix Cache Initialization +# ============================================================================ + +def get_suffix_description(seq: int, text: Optional[str] = None) -> Optional[str]: + """Get description for a suffix by its seq number, with text fallback.""" + if seq is not None: + suffix_class = _suffix_class.get(seq, seq) + desc = SUFFIX_DESCRIPTION.get(suffix_class) + if desc: + return desc + # Fallback: look up by text (for abbreviation suffixes without seq) + if text: + suffix_class = _suffix_text_class.get(text) + if suffix_class: + return SUFFIX_DESCRIPTION.get(suffix_class) + return None + + +def _update_cache(text: str, value: Tuple[str, Optional[Any]], join: bool = False): + """Update suffix cache with a new entry.""" + global _suffix_cache, _suffix_ending_chars + old = _suffix_cache.get(text) + if old is None: + _suffix_cache[text] = [value] + # Track the last character of this suffix for quick filtering + if text: + _suffix_ending_chars.add(text[-1]) + elif join: + _suffix_cache[text].append(value) + else: + _suffix_cache[text] = [value] + + +def get_kana_forms(session: Session, seq: int) -> List[KanaText]: + """ + Get all kana forms for an entry and its conjugations. + + Returns kana text objects for both root and conjugated forms. + """ + # Get kana texts for this seq directly + direct = session.execute( + select(KanaText).where(KanaText.seq == seq) + ).scalars().all() + + # Also get kana texts for conjugations of this seq + conj_seqs = session.execute( + select(Conjugation.seq).where(Conjugation.from_seq == seq) + ).scalars().all() + + if conj_seqs: + indirect = session.execute( + select(KanaText).where(KanaText.seq.in_(conj_seqs)) + ).scalars().all() + else: + indirect = [] + + result = [] + for kt in direct: + kt._conj_type = 'root' + result.append(kt) + + for kt in indirect: + kt._conj_type = 'conj' + result.append(kt) + + return result + + +def get_kana_form(session: Session, seq: int, text: str, conj: Optional[str] = None) -> Optional[KanaText]: + """Get a specific kana form by seq and text.""" + result = session.execute( + select(KanaText).where(and_(KanaText.seq == seq, KanaText.text == text)) + ).scalars().first() + + if result and conj: + result._conj_type = conj + + return result + + +def _load_conjs(session: Session, key: str, seq: int, suffix_class: Optional[str] = None, join: bool = False, include_kanji: bool = False): + """Load all conjugation forms into suffix cache. + + Args: + session: Database session + key: Handler key for suffix matching + seq: Entry sequence number + suffix_class: Optional suffix class (defaults to key) + join: If True, join with existing cache entries + include_kanji: If True, also load kanji text forms + """ + from himotoki.db.models import KanjiText + actual_class = suffix_class or key + for kf in get_kana_forms(session, seq): + _update_cache(kf.text, (key, kf), join=join) + _suffix_class[kf.seq] = actual_class + + # Also load kanji forms if requested + if include_kanji: + kanji_texts = session.execute( + select(KanjiText.text).where(KanjiText.seq == kf.seq) + ).scalars().all() + for kanji_text in kanji_texts: + if kanji_text != kf.text: # Don't duplicate if already kana + _update_cache(kanji_text, (key, kf), join=join) + + +def _load_kf(key: str, kf: KanaText, suffix_class: Optional[str] = None, text: Optional[str] = None, join: bool = False): + """Load a single kana form into suffix cache.""" + actual_text = text or kf.text + actual_class = suffix_class or key + _update_cache(actual_text, (key, kf), join=join) + _suffix_class[kf.seq] = actual_class + + +def _load_abbr(key: str, text: str, join: bool = False, suffix_class: Optional[str] = None): + """Load an abbreviation into suffix cache.""" + _update_cache(text, (key, None), join=join) + if suffix_class: + _suffix_text_class[text] = suffix_class + + +def init_suffixes(session: Session, blocking: bool = True, reset: bool = False): + """ + Initialize the suffix cache. + + This loads all suffix patterns into the cache for efficient lookup. + + Args: + session: Database session + blocking: If True, wait for initialization to complete + reset: If True, force re-initialization + """ + global _suffix_cache, _suffix_class, _suffix_ending_chars, _suffix_initialized + + if _suffix_initialized and not reset: + return + + with _suffix_lock: + if _suffix_initialized and not reset: + return + + _suffix_cache = {} + _suffix_class = {} + _suffix_text_class = {} + _suffix_ending_chars = set() + + # ちゃう (chau) - completion + _load_conjs(session, 'chau', SEQ_CHAU) + _load_conjs(session, 'chau', SEQ_CHIMAU) + + # は particle with ちゃ/じゃ reading + ha_kf = get_kana_form(session, SEQ_WA, 'は') + if ha_kf: + _load_kf('chau', ha_kf, suffix_class='ha', text='ちゃ') + _load_kf('chau', ha_kf, suffix_class='ha', text='じゃ') + + # たい (tai) - want to + _load_conjs(session, 'tai', SEQ_TAI) + + # たそう (tasou) - seem to want to (tai + sou) + tasou_kf = get_kana_form(session, 900000, 'たそう') # Synthetic entry + if tasou_kf: + _load_kf('tai', tasou_kf, suffix_class='tasou') + + # 難い (nikui) - difficult to + _load_conjs(session, 'ren-', SEQ_NIKUI, suffix_class='nikui') + + # おる (oru) - humble progressive + _load_conjs(session, 'te', SEQ_ORU, suffix_class='oru') + + # ある (aru) - result state + _load_conjs(session, 'te', SEQ_ARU, suffix_class='aru') + + # いる (iru) - progressive + for kf in get_kana_forms(session, SEQ_IRU): + tkf = kf.text + if len(tkf) > 1: + _update_cache(tkf, ('teiru+', kf)) + _update_cache(tkf[1:], ('teiru', kf)) + else: + _update_cache(tkf, ('teiru', kf)) + _suffix_class[kf.seq] = 'iru' + + # くる (kuru) - coming to be + _load_conjs(session, 'te', SEQ_KURU, suffix_class='kuru') + + # おく (oku) - in advance + _load_conjs(session, 'te', SEQ_OKU, suffix_class='oku') + _load_conjs(session, 'to', SEQ_TOKU, suffix_class='oku') + + # しまう (shimau) - completion (via chau) + _load_conjs(session, 'te', SEQ_SHIMAU, suffix_class='chau') + + # くれる/もらう/いただく - request forms + _load_conjs(session, 'te+space', SEQ_KURERU, suffix_class='kureru') + _load_conjs(session, 'te+space', SEQ_MORAU, suffix_class='morau') + _load_conjs(session, 'te+space', SEQ_ITADAKU, suffix_class='itadaku') + + # みる (miru) - try doing + _load_conjs(session, 'te+space', SEQ_MIRU, suffix_class='miru') + + # あげる (ageru) - do for someone + _load_conjs(session, 'te+space', SEQ_AGERU, suffix_class='ageru') + + # ほしい (hoshii) - want someone to + _load_conjs(session, 'te+space', SEQ_HOSHII, suffix_class='hoshii') + + # やる (yaru) - do for someone (casual) + _load_conjs(session, 'te+space', SEQ_YARU, suffix_class='yaru') + + # まいる (mairu) - humble go/come (te-form auxiliary) + _load_conjs(session, 'te+space', SEQ_MAIRU, suffix_class='mairu') + + # くださる (kudasaru) - kindly do for (honorific) + _load_conjs(session, 'te+space', SEQ_KUDASARU, suffix_class='kudasaru') + + # さしあげる (sashiageru) - do for (humble/respectful) + _load_conjs(session, 'te+space', SEQ_SASHIAGERU, suffix_class='sashiageru') + + # いく (iku) - going/becoming + for kf in get_kana_forms(session, SEQ_IKU): + tkf = kf.text + if tkf.startswith('い'): + _update_cache(tkf, ('te', kf)) + if len(tkf) > 1: + _update_cache(tkf[1:], ('te', kf)) + _suffix_class[kf.seq] = 'iku' + + # いい (ii) - ok if + ii_kf = get_kana_form(session, SEQ_II, 'いい') + if ii_kf: + _load_kf('teii', ii_kf, suffix_class='ii') + + # もいい - "it's ok if" (て form + もいい) + # This creates the compound ～てもいい pattern + # Try to load from database first, otherwise register directly + moii_kf = get_kana_form(session, SEQ_MOII, 'もいい') + if moii_kf: + _load_kf('teii', moii_kf, suffix_class='ii', text='もいい') + else: + # If the custom entry doesn't exist, register abbreviation for もいい + # This allows ～てもいい to be recognized + _load_abbr('teii', 'もいい', suffix_class='ii') + + # も (mo) - even if + mo_kf = get_kana_form(session, SEQ_MO, 'も') + if mo_kf: + _load_kf('te', mo_kf, suffix_class='mo') + + # ください (kudasai) - please do + kudasai_kf = get_kana_form(session, SEQ_KUDASAI, 'ください', conj='root') + if kudasai_kf: + _load_kf('kudasai', kudasai_kf) + + # する (suru) - make verb from noun + _load_conjs(session, 'suru', SEQ_SURU) + _load_conjs(session, 'suru', SEQ_ITASU, suffix_class='itasu') + _load_conjs(session, 'suru', SEQ_SARERU, suffix_class='sareru') + _load_conjs(session, 'suru', SEQ_SASERU, suffix_class='saseru') + + # そう (sou) - looks like + # Only load そう conjugations, NOT そうにない (SEQ_SOU_NI_NAI = 2141080) + # because そうにない conjugations include そうにな which incorrectly matches + # patterns like 張り裂けそうになる (should be 張り裂けそう + に + なる) + _load_conjs(session, 'sou', SEQ_SOU) + + # NOTE: だろう/でしょう are intentionally NOT loaded as absorbing suffixes. + # They should remain separate tokens (e.g., 神 + だろう, 迷惑 + でしょう + か). + + # すぎる (sugiru) - too much + _load_conjs(session, 'sugiru', 1195970) + + # さ (sa) - -ness + sa_kf = get_kana_form(session, 2029120, 'さ') + if sa_kf: + _load_kf('sa', sa_kf) + + # つつ (tsutsu) - while + tsutsu_kf = get_kana_form(session, 1008120, 'つつ') + if tsutsu_kf: + _load_kf('ren', tsutsu_kf, suffix_class='tsutsu') + _load_conjs(session, 'ren', 2027910, suffix_class='tsutsuaru') + + # ながら (nagara) - while doing + nagara_kf = get_kana_form(session, SEQ_NAGARA, 'ながら') + if nagara_kf: + _load_kf('ren', nagara_kf, suffix_class='nagara') + + # 続ける (tsuzukeru) - to continue (V-続ける pattern) + # Include kanji forms so 続けて matches alongside つづけて + _load_conjs(session, 'ren', 1405800, suffix_class='tsuzukeru', include_kanji=True) + + # っぽい (ppoi) - -ish / tends to + # Only register base form to avoid loading 100+ conjugation forms + # which include short entries like 'ぽ', 'ぽく' causing false matches + ppoi_kf = get_kana_form(session, SEQ_PPOI, 'っぽい') + if ppoi_kf: + _load_kf('ppoi', ppoi_kf) + + # がたい (gatai) - difficult to / hard to + gatai_kf = get_kana_form(session, SEQ_GATAI, 'がたい') + if gatai_kf: + _load_kf('ren', gatai_kf, suffix_class='gatai') + + # 出す (dasu) - to burst out doing / start doing + # Only register base form to avoid kana conflicts (だし, だして etc.) + dasu_kf = get_kana_form(session, SEQ_DASU, 'だす') + if dasu_kf: + _load_kf('ren', dasu_kf, suffix_class='dasu') + _load_kf('ren', dasu_kf, suffix_class='dasu', text='出す') + + # きる (kiru) - to do completely + # Only register base form to avoid short kana conflicts + kiru_kf = get_kana_form(session, SEQ_KIRU, 'きる') + if kiru_kf: + _load_kf('ren', kiru_kf, suffix_class='kiru') + _load_kf('ren', kiru_kf, suffix_class='kiru', text='切る') + + # 方 (kata) - way of doing + kata_kf = get_kana_form(session, SEQ_KATA, 'かた') + if kata_kf: + _load_kf('ren', kata_kf, suffix_class='kata') + _load_kf('ren', kata_kf, suffix_class='kata', text='方') + + # み (mi) - adjective nominalization (-ness) + mi_kf = get_kana_form(session, SEQ_MI, 'み') + if mi_kf: + _load_kf('mi', mi_kf, suffix_class='mi') + + # やすい (yasui) - easy to / likely to + yasui_kf = get_kana_form(session, SEQ_YASUI, 'やすい') + if yasui_kf: + _load_kf('ren', yasui_kf, suffix_class='yasui') + + # まくる (makuru) - to do relentlessly + makuru_kf = get_kana_form(session, SEQ_MAKURU, 'まくる') + if makuru_kf: + _load_kf('ren', makuru_kf, suffix_class='makuru') + + # なおす (naosu) - to redo / do again + naosu_kf = get_kana_form(session, SEQ_NAOSU, 'なおす') + if naosu_kf: + _load_kf('ren', naosu_kf, suffix_class='naosu') + + # そこなう (sokonau) - to fail to do + sokonau_kf = get_kana_form(session, SEQ_SOKONAU, 'そこなう') + if sokonau_kf: + _load_kf('ren', sokonau_kf, suffix_class='sokonau') + + # わすれる (wasureru) - to forget to do + wasureru_kf = get_kana_form(session, SEQ_WASURERU, 'わすれる') + if wasureru_kf: + _load_kf('ren', wasureru_kf, suffix_class='wasureru') + + # おえる (oeru) - to finish doing + oeru_kf = get_kana_form(session, SEQ_OERU, 'おえる') + if oeru_kf: + _load_kf('ren', oeru_kf, suffix_class='oeru') + + # づらい (zurai) - difficult to + zurai_kf = get_kana_form(session, SEQ_ZURAI, 'づらい') + if zurai_kf: + _load_kf('ren', zurai_kf, suffix_class='zurai') + + # ぎみ (gimi) - tending to / -ish + gimi_kf = get_kana_form(session, SEQ_GIMI, 'ぎみ') + if gimi_kf: + _load_kf('ren', gimi_kf, suffix_class='gimi') + + # っぱなし (ppanashi) - left doing + ppanashi_kf = get_kana_form(session, SEQ_PPANASHI, 'っぱなし') + if ppanashi_kf: + _load_kf('ren', ppanashi_kf, suffix_class='ppanashi') + + # たち (tachi) - plural suffix + tachi_kf = get_kana_form(session, SEQ_TACHI, 'たち') + if tachi_kf: + _load_kf('tachi', tachi_kf) + + # 合う (au) - mutual/reciprocal compound verb + au_kf = get_kana_form(session, SEQ_AU, 'あう') + if au_kf: + _load_kf('ren', au_kf, suffix_class='au') + _load_kf('ren', au_kf, suffix_class='au', text='合う') + + # 込む (komu) - into/thoroughly compound verb + komu_kf = get_kana_form(session, SEQ_KOMU, 'こむ') + if komu_kf: + _load_kf('ren', komu_kf, suffix_class='komu') + _load_kf('ren', komu_kf, suffix_class='komu', text='込む') + + # 放題 (houdai) - as much as one likes + houdai_kf = get_kana_form(session, SEQ_HOUDAI, 'ほうだい') + if houdai_kf: + _load_kf('ren', houdai_kf, suffix_class='houdai') + _load_kf('ren', houdai_kf, suffix_class='houdai', text='放題') + + # 終わる (owaru) - to finish doing + owaru_kf = get_kana_form(session, SEQ_OWARU, 'おわる') + if owaru_kf: + _load_kf('ren+', owaru_kf, suffix_class='owaru') + _load_kf('ren+', owaru_kf, suffix_class='owaru', text='終わる') + + # 始める (hajimeru) - to start doing + hajimeru_kf = get_kana_form(session, SEQ_HAJIMERU, 'はじめる') + if hajimeru_kf: + _load_kf('ren+', hajimeru_kf, suffix_class='hajimeru') + _load_kf('ren+', hajimeru_kf, suffix_class='hajimeru', text='始める') + + # つける (tsukeru) - to be accustomed to doing + tsukeru_kf = get_kana_form(session, SEQ_TSUKERU, 'つける') + if tsukeru_kf: + _load_kf('ren', tsukeru_kf, suffix_class='tsukeru') + + # うる (uru) - can + uru_kf = get_kana_form(session, 1454500, 'うる') + if uru_kf: + _load_kf('ren', uru_kf, suffix_class='uru') + + # なく (naku) - negative + naku_kf = session.execute( + select(KanaText) + .join(Conjugation, KanaText.seq == Conjugation.seq) + .where(and_(KanaText.text == 'なく', Conjugation.from_seq == 1529520)) + ).scalars().first() + if naku_kf: + _load_kf('neg', naku_kf, suffix_class='nai') + + # ないで (naide) - negative te-form "without doing" / "don't" + naide_kf = get_kana_form(session, 2258690, 'ないで') + if naide_kf: + _load_kf('neg', naide_kf, suffix_class='naide') + + # なる (naru) - become + _load_conjs(session, 'adv', 1375610, suffix_class='naru') + + # やがる (yagaru) - disdain + _load_conjs(session, 'teren', 1012740, suffix_class='yagaru') + + # ら (ra) - plural + ra_kf = get_kana_form(session, 2067770, 'ら') + if ra_kf: + _load_kf('ra', ra_kf) + + # らしい (rashii) - seems like + _load_conjs(session, 'rashii', 1013240) + + # です / でした (desu) - formal copula forms + desu_kf = get_kana_form(session, 1628500, 'です') + if desu_kf: + _load_kf('desu', desu_kf) + deshita_kf = get_kana_form(session, 10044689, 'でした') + if deshita_kf: + _load_kf('desu', deshita_kf, text='でした') + + # とする (tosuru) - try to + _load_conjs(session, 'tosuru', 2136890) + + # くらい (kurai) - about/approximately + kurai_kf = get_kana_form(session, 1154340, 'くらい') + if kurai_kf: + _load_kf('kurai', kurai_kf) + gurai_kf = get_kana_form(session, 1154340, 'ぐらい') + if gurai_kf: + _load_kf('kurai', gurai_kf) + + # がる (garu) - feel + _load_conjs(session, 'garu', 1631750) + + # がち (gachi) - tend to + gachi_kf = get_kana_form(session, 2016470, 'がち') + if gachi_kf: + _load_kf('ren', gachi_kf, suffix_class='gachi') + + # げ (ge) - -ish + ge_kf = get_kana_form(session, 2006580, 'げ') + if ge_kf: + _load_kf('iadj', ge_kf) + + # め (me) - somewhat + me_kf = get_kana_form(session, 1604890, 'め') + if me_kf: + _load_kf('iadj', me_kf, suffix_class='me') + + # がい (gai) - worth + gai_kf = get_kana_form(session, 2606690, 'がい') + if gai_kf: + _load_kf('ren-', gai_kf, suffix_class='gai') + + # Abbreviations + _load_abbr('nai', 'ねえ') + _load_abbr('nai', 'ねぇ') + _load_abbr('nai', 'ねー') + _load_abbr('nai-x', 'ず') + _load_abbr('nai-x', 'ざる') + _load_abbr('nai-x', 'ぬ') + _load_abbr('nai-n', 'ん') + + _load_abbr('nakereba', 'なきゃ', suffix_class='nakereba') + _load_abbr('nakereba', 'なくちゃ', suffix_class='nakereba') + + _load_abbr('teba', 'ちゃ', join=True) # つ + _load_abbr('reba', 'りゃ') # る + _load_abbr('keba', 'きゃ') # く + _load_abbr('geba', 'ぎゃ') # ぐ + _load_abbr('neba', 'にゃ') # ぬ + _load_abbr('beba', 'びゃ') # ぶ + _load_abbr('meba', 'みゃ') # む + _load_abbr('seba', 'しゃ') # す + + _load_abbr('shimashou', 'ましょ', suffix_class='shimashou') + _load_abbr('dewanai', 'じゃない') + + _load_abbr('ii', 'ええ') + + # な-adjective て-form: 静かで, 元気で (copula て-form) + _load_abbr('nade', 'で', suffix_class='nade') + + _suffix_initialized = True + + +def is_suffix_cache_ready() -> bool: + """Check if suffix cache is initialized.""" + return _suffix_initialized + + +def could_have_suffix(word: str) -> bool: + """ + Quick check if a word could possibly have a suffix. + + Returns False if the word doesn't end with any character that + appears at the end of known suffixes. This is a cheap O(1) filter. + + Args: + word: Word to check + + Returns: + True if a suffix could potentially match, False if definitely not + """ + if not _suffix_initialized or len(word) < 2: + return False + # Check if word ends with any character that suffixes end with + return word[-1] in _suffix_ending_chars + + +# ============================================================================ +# Suffix Lookup Functions +# ============================================================================ + +def get_suffix_map(session: Session, text: str) -> Dict[int, List[Tuple[str, str, Optional[KanaText]]]]: + """ + Create a suffix map for the given text. + + Maps end positions to list of (suffix, keyword, kana_form) tuples. + + Args: + session: Database session + text: Text to create suffix map for + + Returns: + Dictionary mapping end position to suffix entries + """ + init_suffixes(session) + + result: Dict[int, List[Tuple[str, str, Optional[KanaText]]]] = {} + + for start in range(len(text)): + for end in range(start + 1, len(text) + 1): + substr = text[start:end] + vals = _suffix_cache.get(substr) + if vals: + for keyword, kf in vals: + if end not in result: + result[end] = [] + result[end].append((substr, keyword, kf)) + + return result + + +def get_suffixes(session: Session, word: str) -> List[Tuple[str, str, Optional[KanaText]]]: + """ + Get all suffix matches for a word. + + Searches from the end of the word backwards. + + Args: + session: Database session + word: Word to find suffixes for + + Returns: + List of (suffix, keyword, kana_form) tuples + """ + init_suffixes(session) + + results = [] + for start in range(len(word) - 1, 0, -1): + substr = word[start:] + vals = _suffix_cache.get(substr) + if vals: + for keyword, kf in vals: + results.append((substr, keyword, kf)) + + return results + + +# ============================================================================ +# Suffix Matching +# ============================================================================ + +# List of suffix types where only unique matches are allowed +SUFFIX_UNIQUE_ONLY: Set[str] = { + 'ra', 'mo', 'nikui', 'gai', 'nai-n', 'dewanai', + 'eba', 'teba', 'reba', 'keba', 'geba', 'neba', 'beba', 'meba', 'seba', 'ii', +} + +# Words that should NEVER be decomposed into root+suffix compounds. +# e.g., ため should not be split into た (adj stem of たい) + め (suffix). +BLOCKED_SUFFIX_WORDS: Set[str] = { + 'ため', # "for/in order to" - not た+め + 'だめ', # "no good" - not だ+め + 'さげ', # "lowering" - not さ+げ + 'もまず', # も+まず (also+first) - not 揉む+ず (not-rub) +} + +# Abbreviation suffix types - these create compounds that should be scored +# without using the abbreviated compound's mora length. +# Matches Ichiran's def-abbr-suffix handlers which create proxy-text objects. +ABBREVIATION_SUFFIXES: Set[str] = { + 'nai', 'nai-x', 'nai-n', 'nakereba', 'shimashou', 'dewanai', + 'teba', 'reba', 'keba', 'geba', 'neba', 'beba', 'meba', 'seba', 'ii', +} + +# Stem lengths to remove from kana for abbreviation suffixes +# These correspond to the stem parameter in ichiran's def-abbr-suffix +# e.g., 2 means remove 2 kana chars from end before adding suffix +# nai-x (ず/ざる/ぬ): stem=2 removes ない → あきらめない → あきらめ + ず = あきらめず +# nakereba (なきゃ): stem=4 removes なければ → しなければ → し + なきゃ = しなきゃ +ABBREVIATION_STEMS: Dict[str, int] = { + 'nai': 2, # ねー etc - ない → ね (remove 2: ない) + 'nai-x': 2, # ず, ざる, ぬ - ない → stem (remove 2: ない) + 'nai-n': 2, # ん contraction - ない → な (remove 2: ない) + 'nakereba': 4, # なきゃ/なくちゃ - なければ → stem (remove 4: なければ) + 'shimashou': 4, # ましょ - ましょう → stem (remove 4: ましょう) + 'dewanai': 4, # じゃない - ではない → stem (remove 4: ではない) + 'teba': 2, # ちゃ - てば → stem (remove 2) + 'reba': 2, # りゃ - れば → stem (remove 2) + 'keba': 2, # きゃ - けば → stem (remove 2) + 'geba': 2, # ぎゃ - げば → stem (remove 2) + 'neba': 2, # にゃ - ねば → stem (remove 2) + 'beba': 2, # びゃ - べば → stem (remove 2) + 'meba': 2, # みゃ - めば → stem (remove 2) + 'seba': 2, # しゃ - せば → stem (remove 2) + 'ii': 2, # ええ - いい → stem (remove 2) +} + + +def match_unique(suffix_class: str, matches: List[Any]) -> bool: + """ + Check if only unique matches are allowed for this suffix class. + + Some suffix patterns should only match when no other word exists. + """ + return suffix_class in SUFFIX_UNIQUE_ONLY + + +def find_word_suffix( + session: Session, + word: str, + matches: Optional[List[Any]] = None, + suffix_map: Optional[Dict] = None, + next_end: Optional[int] = None, + depth: int = 0, +) -> List[Any]: + """ + Find suffix matches for a word. + + This tries to match grammatical suffixes at the end of the word + and return compound word objects. + + Ports ichiran's find-word-suffix from dict-grammar.lisp lines 692-706. + + Args: + session: Database session + word: Word to find suffix for + matches: Existing matches (for uniqueness checking) + suffix_map: Pre-computed suffix map (optional) + next_end: End position in source text (optional) + depth: Current recursion depth for nested compounds + + Returns: + List of compound word matches + """ + # Block known words that should never be decomposed into root+suffix + if word in BLOCKED_SUFFIX_WORDS: + return [] + + # Guard against excessive recursion using both explicit depth and context var + current_depth = _current_suffix_depth.get() + effective_depth = max(depth, current_depth) + if effective_depth >= MAX_SUFFIX_DEPTH: + return [] + + # Set context var for handlers that may call find_word_suffix recursively + token = _current_suffix_depth.set(effective_depth + 1) + + try: + from himotoki.lookup import adjoin_word, WordMatch + + init_suffixes(session) + + # Get suffixes from map or by direct lookup + if suffix_map and next_end: + suffixes = suffix_map.get(next_end, []) + else: + suffixes = get_suffixes(session, word) + + results = [] + + for suffix, keyword, kf in suffixes: + # Check uniqueness constraint + suffix_class = _suffix_class.get(kf.seq if kf else None, keyword) + if matches and match_unique(suffix_class, matches): + continue + + offset = len(word) - len(suffix) + if offset <= 0: + continue + + root = word[:offset] + + # Get suffix function based on keyword + suffix_fn = SUFFIX_HANDLERS.get(keyword) + if not suffix_fn: + continue + + # Call suffix handler to get primary words + primary_words = suffix_fn(session, root, suffix, kf) + + for pw in primary_words: + if pw is None: + continue + + # Create suffix word for adjoin + # Get conjugation IDs for the suffix word if it's a conjugated form + suffix_conj_ids = None + if kf and hasattr(kf, '_conj_type') and kf._conj_type == 'conj': + # This is a conjugated form - get the conjugation IDs + from himotoki.db.models import Conjugation + conj_query = select(Conjugation.id).where(Conjugation.seq == kf.seq) + suffix_conj_ids = list(session.execute(conj_query).scalars().all()) + + if kf: + suffix_word = WordMatch(reading=kf, conjugations=suffix_conj_ids) + else: + # Create a placeholder for the suffix (abbreviation case) + # For abbreviations like もいい, we create a minimal placeholder + # that looks like a KanaText but without a database entry + class PlaceholderReading: + def __init__(self, text): + self.text = text + self.seq = None + self.ord = 0 + self.common = None + placeholder = PlaceholderReading(suffix) + suffix_word = WordMatch(reading=placeholder) + + # Use adjoin_word to create compound (following ichiran's pattern) + # Score is determined by the suffix handler configuration + # For 'sou' suffix, use conditional scoring based on root + if keyword in ('sou', 'sou+'): + score_mod = get_sou_score(root) + else: + score_mod = SUFFIX_SCORES.get(keyword, 0) + connector = SUFFIX_CONNECTORS.get(keyword, '') + + # Check if this is an abbreviation suffix + is_abbrev = keyword in ABBREVIATION_SUFFIXES + + # Get kana for the compound + # For primary word: get kana from reading, look up if kanji + def get_word_kana(w): + from himotoki.lookup import CompoundWord + # For CompoundWord, return the compound's kana directly + if isinstance(w, CompoundWord): + return w.kana + if hasattr(w, 'reading'): + reading = w.reading + # Check if it's a kanji reading - look up kana + if hasattr(reading, 'seq') and hasattr(reading, 'text'): + # Use the reading's ord to get the matching kana + # (e.g., 食べないで ord=1 → たべないで ord=1, not たべなくて ord=0) + reading_ord = getattr(reading, 'ord', None) + if reading_ord is not None and isinstance(reading, KanjiText): + kana_result = session.execute( + select(KanaText.text) + .where(and_(KanaText.seq == reading.seq, KanaText.ord == reading_ord)) + ).scalars().first() + if kana_result: + return kana_result + # Fallback: get first kana for this seq + kana_result = session.execute( + select(KanaText.text) + .where(KanaText.seq == reading.seq) + ).scalars().first() + if kana_result: + return kana_result + # If no kana found, assume text is already kana + return reading.text + return w.text if hasattr(w, 'text') else '' + + primary_kana = get_word_kana(pw) + suffix_kana = kf.text if kf else suffix + + # For abbreviation suffixes, remove stem characters from primary kana + # This mirrors ichiran's def-abbr-suffix destem behavior + abbr_stem = ABBREVIATION_STEMS.get(keyword, 0) + if abbr_stem > 0 and len(primary_kana) > abbr_stem: + primary_kana = primary_kana[:-abbr_stem] + + # For contraction suffixes (chau, to), the handler reconstructs + # the te-form (root + て/で) for lookup. The primary word kana + # includes trailing て/で but in the surface text, that て/で + # has been contracted (ちゃ, じゃ, と, ど). Strip the trailing + # て/で from primary kana to avoid kana inflation. + # Also use surface suffix text for kana (e.g., ちゃ not は). + if keyword in ('chau', 'to'): + if primary_kana.endswith('て') or primary_kana.endswith('で'): + primary_kana = primary_kana[:-1] + suffix_kana = suffix + + # For teiru (contracted, without leading い), the surface suffix + # is shorter than kf.text (e.g., る vs いる). Use the actual + # suffix text for kana to match the contracted pronunciation. + if keyword == 'teiru' and kf and len(suffix) < len(kf.text): + suffix_kana = suffix + + # Include connector in kana (e.g., space for suru, kudasai, te+space) + compound_kana = primary_kana + connector + suffix_kana + + compound = adjoin_word( + pw, + suffix_word, + text=word, + kana=compound_kana, + score_mod=score_mod, + is_abbrev=is_abbrev, + ) + results.append(compound) + + return results + finally: + # Reset context var to previous value + _current_suffix_depth.reset(token) + + +# Suffix scores - from def-simple-suffix definitions in ichiran's dict-grammar.lisp +SUFFIX_SCORES: Dict[str, float] = { + 'tai': 5, + 'ren': 5, + 'ren+': 10, # Higher score for longer compound verb suffixes (3+ chars) + 'ren-': 0, + 'neg': 5, + 'te': 0, + 'teiru': 3, + 'teiru+': 6, + 'te+space': 3, + 'teren': 4, + 'teii': 1, + 'chau': 5, + 'to': 0, + 'suru': 5, + 'sou': 60, # Default, but overridden by get_sou_score for specific roots + 'sou+': 1, + 'adv': 1, + 'sugiru': 5, + 'sa': 2, + 'iadj': 1, + 'mi': 1, + 'garu': 0, + 'ra': 1, + 'rashii': 3, + 'ppoi': 3, + 'tachi': 3, + 'desu': 200, + 'tosuru': 3, + 'kurai': 3, + 'nai': 5, + 'kudasai': 360, + 'nade': 3, +} + +# Suffix connectors - space between root and suffix in kana +SUFFIX_CONNECTORS: Dict[str, str] = { + 'suru': ' ', + 'kudasai': ' ', + 'te+space': ' ', + 'teii': ' ', +} + + +def get_sou_score(root: str) -> float: + """ + Get the score_mod for そう suffix based on the root. + + Ports ichiran's conditional scoring for suffix-sou: + - root "から" → 40 + - root "い" → 0 + - root "出来" → 100 + - otherwise → 60 + + Additional adjustment: reduce score for short hiragana-only roots + that could be confused with verb continuative forms (e.g., いた, よさ). + This prevents incorrect splits like 言+いたそう when 言いたそう exists. + """ + from himotoki.characters import count_char_class + + if root == 'から': + return 40 + elif root == 'い': + return 0 + elif root == '出来': + return 100 + else: + # For short hiragana-only roots, reduce the score to prevent + # incorrect splits with verb continuatives + if len(root) <= 2 and count_char_class(root, 'kanji') == 0: + return 10 # Reduced from 60 to lower priority + return 60 + + +# ============================================================================ +# Suffix Handler Functions +# ============================================================================ + +def find_word_with_conj_type(session: Session, word: str, *conj_types: int) -> List[Any]: + """Import and call find_word_with_conj_type from lookup.""" + from himotoki.lookup import find_word_with_conj_type as _find_word_with_conj_type + return _find_word_with_conj_type(session, word, *conj_types) + + +def find_word_with_pos(session: Session, word: str, *posi: str) -> List[Any]: + """Find words with specific part-of-speech tags.""" + from himotoki.lookup import find_word + from himotoki.db.models import SenseProp + from sqlalchemy import select, and_ + + words = find_word(session, word) + results = [] + + for w in words: + # Check if any sense has the required POS + has_pos = session.execute( + select(SenseProp) + .where(and_( + SenseProp.seq == w.seq, + SenseProp.tag == 'pos', + SenseProp.text.in_(posi) + )) + ).scalars().first() + + if has_pos: + results.append(w) + + return results + + +def find_word_with_neg_prop(session: Session, word: str) -> List[Any]: + """Find words with negative conjugation property.""" + from himotoki.lookup import find_word_with_conj_prop + + def filter_fn(cdata): + if cdata.prop and hasattr(cdata.prop, 'neg'): + return cdata.prop.neg + return False + + return find_word_with_conj_prop(session, word, filter_fn) + + + +from himotoki.grammar.suffix_handlers import SUFFIX_HANDLERS diff --git a/himotoki/grammar/synergies.py b/himotoki/grammar/synergies.py new file mode 100644 index 0000000..fbf3e7e --- /dev/null +++ b/himotoki/grammar/synergies.py @@ -0,0 +1,313 @@ +""" +Synergies and Segfilters runtime API for himotoki. +Ports ichiran's dict-grammar.lisp synergy and segfilter functionality. +""" + +from dataclasses import dataclass +from typing import Optional, List, Dict, Tuple, Union, Any, Callable + +from himotoki.constants import NOUN_PARTICLES + +from himotoki.grammar.synergy_filters import ( + cached_filter, + filter_is_noun, + filter_is_pos, + filter_in_seq_set, + filter_in_seq_set_simple, + filter_is_conjugation, + filter_is_compound_end, + filter_is_compound_end_text, + filter_short_kana, +) + +# ============================================================================ +# Synergy Data Structure +# ============================================================================ + +@dataclass(slots=True) +class Synergy: + """Represents a synergy between two adjacent segments.""" + description: str + connector: str # Space or empty string between words + score: int + start: int # Position where synergy starts + end: int # Position where synergy ends + + +def get_segment_score_synergy(syn: Synergy) -> int: + """Get the score value from a synergy.""" + return syn.score + + +# ============================================================================ +# Synergy List and Registration +# ============================================================================ + +_synergy_list: List[Callable] = [] + + +def register_synergy(func: Callable): + """Register a synergy function.""" + _synergy_list.append(func) + return func + + +def def_generic_synergy( + name: str, + filter_left: Callable, + filter_right: Callable, + description: str, + score: Union[int, Callable], + connector: str = " ", +): + """ + Define a generic synergy between two segment lists. + """ + def synergy_fn(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + start = seg_list_left.end + end = seg_list_right.start + + # Must be adjacent + if start != end: + return [] + + left_segments = [s for s in seg_list_left.segments if filter_left(s)] + right_segments = [s for s in seg_list_right.segments if filter_right(s)] + + if not left_segments or not right_segments: + return [] + + # Calculate score + if callable(score): + actual_score = score(seg_list_left, seg_list_right) + else: + actual_score = score + + # Create synergy + syn = Synergy( + description=description, + connector=connector, + score=actual_score, + start=start, + end=end, + ) + + # Return modified segment lists with synergy + from himotoki.lookup import SegmentList + new_left = SegmentList( + segments=left_segments, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_segments, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_fn) + return synergy_fn + +def get_synergies(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """ + Get all synergies between two segment lists. + + Returns list of (new_right, synergy, new_left) tuples. + """ + results = [] + for fn in _synergy_list: + results.extend(fn(seg_list_left, seg_list_right)) + return results +# ============================================================================ +# Penalty List and Registration +# ============================================================================ + +_penalty_list: List[Callable] = [] + + +def register_penalty(func: Callable): + """Register a penalty function.""" + _penalty_list.append(func) + return func + + +def def_generic_penalty( + name: str, + test_left: Callable, + test_right: Callable, + description: str, + score: int, + serial: bool = True, + connector: str = " ", +): + """ + Define a generic penalty between two segment lists. + """ + def penalty_fn(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: + start = seg_list_left.end + end = seg_list_right.start + + # Check serial requirement + if serial and start != end: + return None + + if not test_left(seg_list_left): + return None + if not test_right(seg_list_right): + return None + + return Synergy( + description=description, + connector=connector, + score=score, + start=start, + end=end, + ) + + register_penalty(penalty_fn) + return penalty_fn + +def get_penalties(seg_list_left: Any, seg_list_right: Any) -> List[Any]: + """ + Get penalties between two segment lists. + + Returns [seg_right, penalty, seg_left] if penalty applies, + otherwise [seg_right, seg_left]. + """ + for fn in _penalty_list: + penalty = fn(seg_list_left, seg_list_right) + if penalty: + return [seg_list_right, penalty, seg_list_left] + + return [seg_list_right, seg_list_left] +# ============================================================================ +# Segfilter List and Registration +# ============================================================================ + +_segfilter_list: List[Callable] = [] + + +def register_segfilter(func: Callable): + """Register a segfilter function.""" + _segfilter_list.append(func) + return func + + +def def_segfilter_must_follow( + name: str, + filter_left: Callable, + filter_right: Callable, + allow_first: bool = False, +): + """ + Define a segfilter where filter_right MUST follow filter_left. + + If filter_right matches but filter_left doesn't (and we're not at the start), + the matching segments are removed. + """ + def classify(filter_fn: Callable, items: List) -> Tuple[List, List]: + satisfies = [] + contradicts = [] + for item in items: + if filter_fn(item): + satisfies.append(item) + else: + contradicts.append(item) + return satisfies, contradicts + + def segfilter_fn(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + from himotoki.lookup import SegmentList + + satisfies_right, contradicts_right = classify(filter_right, seg_list_right.segments) + + # If nothing satisfies filter_right, pass through + if not satisfies_right: + return [(seg_list_left, seg_list_right)] + + # If first position and allowed, pass through + if allow_first and seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # If not adjacent, only allow contradicts + if seg_list_left is None or seg_list_left.end != seg_list_right.start: + if contradicts_right: + new_right = SegmentList( + segments=contradicts_right, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + return [(seg_list_left, new_right)] + return [] + + # Check left side + satisfies_left, contradicts_left = classify(filter_left, seg_list_left.segments) + + results = [] + + # If left has contradicts, allow those with right contradicts + if contradicts_left and contradicts_right: + results.append(( + seg_list_left, + SegmentList( + segments=contradicts_right, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )) + + # If left satisfies, allow with right satisfies + if satisfies_left: + results.append(( + SegmentList( + segments=satisfies_left, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ), + SegmentList( + segments=satisfies_right, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )) + + if not results and contradicts_right: + results.append(( + seg_list_left, + SegmentList( + segments=contradicts_right, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )) + + return results if results else [(seg_list_left, seg_list_right)] + + register_segfilter(segfilter_fn) + return segfilter_fn +def apply_segfilters(seg_left: Optional[Any], seg_right: Any) -> List[Tuple]: + """ + Apply all segfilters to a pair of segment lists. + + Returns list of (seg_left, seg_right) pairs that pass all filters. + """ + splits = [(seg_left, seg_right)] + + for segfilter in _segfilter_list: + new_splits = [] + for seg_l, seg_r in splits: + new_splits.extend(segfilter(seg_l, seg_r)) + splits = new_splits + + return splits + +# Load rule registrations +import himotoki.grammar.synergy_rules # noqa: F401 diff --git a/himotoki/grammar/synergy_filters.py b/himotoki/grammar/synergy_filters.py new file mode 100644 index 0000000..6096ae4 --- /dev/null +++ b/himotoki/grammar/synergy_filters.py @@ -0,0 +1,261 @@ +""" +Filter helpers and caching for synergy/penalty/segfilter matching. +""" + +from typing import Optional, List, Dict, Any, Callable + +# ============================================================================ +# Filter Caching System +# ============================================================================ + +# Global filter ID counter for unique identification +_filter_id_counter = 0 + + +def _get_next_filter_id() -> int: + """Get a unique filter ID.""" + global _filter_id_counter + _filter_id_counter += 1 + return _filter_id_counter + + +def cached_filter(filter_fn: Callable) -> Callable: + """ + Wrap a filter function with segment-level caching. + + Each filter gets a unique ID, and results are cached on the segment + to avoid re-running the same filter on the same segment. + """ + filter_id = _get_next_filter_id() + + def _cached_filter(segment: Any) -> bool: + # Try to get cached result + if hasattr(segment, 'get_filter_result'): + cached = segment.get_filter_result(filter_id) + if cached is not None: + return cached + + # Compute and cache result + result = filter_fn(segment) + + if hasattr(segment, 'set_filter_result'): + segment.set_filter_result(filter_id, result) + + return result + + # Preserve the filter's identity for debugging + _cached_filter.__name__ = getattr(filter_fn, '__name__', 'cached_filter') + _cached_filter._filter_id = filter_id + + return _cached_filter + + +# ============================================================================ +# Filter Helpers +# ============================================================================ + +def _filter_is_noun_impl(segment: Any) -> bool: + """Check if segment is a noun (implementation).""" + info = getattr(segment, 'info', {}) + kpcl = info.get('kpcl', [False, False, False, False]) + if len(kpcl) < 4: + kpcl = kpcl + [False] * (4 - len(kpcl)) + k, p, c, l = kpcl + + posi = info.get('posi', []) + noun_pos = {'n', 'n-adv', 'n-t', 'adj-na', 'n-suf', 'pn'} + + if (l or k or (p and c)) and noun_pos.intersection(posi): + return True + + # Check for counter + word = getattr(segment, 'word', None) + if word and hasattr(word, '__class__') and 'counter' in word.__class__.__name__.lower(): + return bool(info.get('seq_set')) + + return False + + +# Create cached version of filter_is_noun +filter_is_noun = cached_filter(_filter_is_noun_impl) + + +# Cache for filter_is_pos filters to avoid creating duplicates +_pos_filter_cache: Dict[frozenset, Callable] = {} + + +def filter_is_pos(*pos_list: str): + """Create filter for specific parts of speech (cached).""" + pos_key = frozenset(pos_list) + if pos_key in _pos_filter_cache: + return _pos_filter_cache[pos_key] + + pos_set = set(pos_list) + + def _filter(segment: Any) -> bool: + info = getattr(segment, 'info', {}) + posi = info.get('posi', []) + return bool(pos_set.intersection(posi)) + + result = cached_filter(_filter) + _pos_filter_cache[pos_key] = result + return result + + +# Cache for filter_in_seq_set filters +_seq_filter_cache: Dict[frozenset, Callable] = {} + + +def filter_in_seq_set(*seqs: int): + """Create filter for specific sequence numbers (cached).""" + seq_key = frozenset(seqs) + if seq_key in _seq_filter_cache: + return _seq_filter_cache[seq_key] + + seq_set = set(seqs) + + def _filter(segment: Any) -> bool: + info = getattr(segment, 'info', {}) + segment_seqs = info.get('seq_set', []) + return bool(seq_set.intersection(segment_seqs)) + + result = cached_filter(_filter) + _seq_filter_cache[seq_key] = result + return result + + +# Cache for filter_in_seq_set_simple filters +_seq_simple_filter_cache: Dict[frozenset, Callable] = {} + + +def filter_in_seq_set_simple(*seqs: int): + """Filter for seqs, checking that word is not compound (cached).""" + seq_key = frozenset(seqs) + if seq_key in _seq_simple_filter_cache: + return _seq_simple_filter_cache[seq_key] + + seq_set = set(seqs) + + def _filter(segment: Any) -> bool: + word = getattr(segment, 'word', None) + if not word: + return False + seq = getattr(word, 'seq', None) + if isinstance(seq, list): + return False + info = getattr(segment, 'info', {}) + segment_seqs = info.get('seq_set', []) + return bool(seq_set.intersection(segment_seqs)) + + result = cached_filter(_filter) + _seq_simple_filter_cache[seq_key] = result + return result + + +# Cache for filter_is_conjugation filters +_conj_filter_cache: Dict[int, Callable] = {} + + +def filter_is_conjugation(conj_type: int): + """Create filter for specific conjugation type (cached).""" + if conj_type in _conj_filter_cache: + return _conj_filter_cache[conj_type] + + def _filter(segment: Any) -> bool: + info = getattr(segment, 'info', {}) + conj = info.get('conj', []) + for cdata in conj: + if hasattr(cdata, 'prop') and cdata.prop: + if getattr(cdata.prop, 'conj_type', None) == conj_type: + return True + return False + + result = cached_filter(_filter) + _conj_filter_cache[conj_type] = result + return result + + +# Cache for filter_is_compound_end filters +_compound_end_filter_cache: Dict[frozenset, Callable] = {} + + +def filter_is_compound_end(*seqs: int): + """Filter for compound words ending with specific seqs (cached).""" + seq_key = frozenset(seqs) + if seq_key in _compound_end_filter_cache: + return _compound_end_filter_cache[seq_key] + + seq_set = set(seqs) + + def _filter(segment: Any) -> bool: + word = getattr(segment, 'word', None) + if not word: + return False + seq = getattr(word, 'seq', None) + if isinstance(seq, list) and seq: + return seq[-1] in seq_set + return False + + result = cached_filter(_filter) + _compound_end_filter_cache[seq_key] = result + return result + + +# Cache for filter_is_compound_end_text filters +_compound_end_text_filter_cache: Dict[frozenset, Callable] = {} + + +def filter_is_compound_end_text(*texts: str): + """Filter for compound words ending with specific texts (cached).""" + text_key = frozenset(texts) + if text_key in _compound_end_text_filter_cache: + return _compound_end_text_filter_cache[text_key] + + text_set = set(texts) + + def _filter(segment: Any) -> bool: + word = getattr(segment, 'word', None) + if not word: + return False + seq = getattr(word, 'seq', None) + if not isinstance(seq, list): + return False + words = getattr(word, 'words', []) + if words: + last_word = words[-1] + text = getattr(last_word, 'text', '') + return text in text_set + return False + + result = cached_filter(_filter) + _compound_end_text_filter_cache[text_key] = result + return result + + +def filter_short_kana(length: int, except_list: Optional[List[str]] = None): + """Filter for short kana words.""" + except_set = set(except_list) if except_list else set() + + def _filter(segment_list: Any) -> bool: + segments = getattr(segment_list, 'segments', []) + if not segments: + return False + seg = segments[0] + + seg_len = segment_list.end - segment_list.start + if seg_len > length: + return False + + info = getattr(seg, 'info', {}) + kpcl = info.get('kpcl', [False, False, False, False]) + if kpcl and kpcl[0]: # Has kanji + return False + + text = getattr(seg, 'text', '') or getattr(seg.word, 'text', '') + if text in except_set: + return False + + return True + + return _filter + diff --git a/himotoki/grammar/synergy_rules.py b/himotoki/grammar/synergy_rules.py new file mode 100644 index 0000000..e2e618b --- /dev/null +++ b/himotoki/grammar/synergy_rules.py @@ -0,0 +1,2078 @@ +""" +Synergy, penalty, and segfilter rule registrations. +""" + +from typing import Optional, List, Dict, Tuple, Union, Any, Callable + +from sqlalchemy import select, and_, or_ +from sqlalchemy.orm import Session + +from himotoki.db.models import KanjiText, KanaText +from himotoki.constants import ( + SEQ_WA, SEQ_GA, SEQ_NI, SEQ_DE, SEQ_HE, SEQ_WO, SEQ_NO, SEQ_TO, SEQ_MO, SEQ_YA, SEQ_KA, + SEQ_NIHA, SEQ_TOHA, SEQ_TOKA, SEQ_TOSHITE, SEQ_DESAE, + SEQ_DAKE, SEQ_GORO, SEQ_MADE, SEQ_NADO, SEQ_NOMI, SEQ_SAE, SEQ_TTE, SEQ_KARA, SEQ_NITOTTE, + SEQ_SURU, SEQ_IRU, SEQ_KURU, + SEQ_MAE_NOUN, SEQ_HOU_NOUN, SEQ_MEN_NOUN, + SEQ_HITO_NOUN, SEQ_NAKA_NOUN, + SEQ_TOMARU, SEQ_TODOMARU, + SEQ_KARAI, SEQ_TSURAI, + SEQ_NITSURE, SEQ_OSUSUME, + NOUN_PARTICLES, +) + +from himotoki.grammar.synergy_filters import ( + filter_is_noun, + filter_is_pos, + filter_in_seq_set, + filter_in_seq_set_simple, + filter_is_conjugation, + filter_is_compound_end, + filter_is_compound_end_text, + filter_short_kana, +) +from himotoki.grammar.synergies import ( + Synergy, + register_synergy, + def_generic_synergy, + register_penalty, + def_generic_penalty, + register_segfilter, + def_segfilter_must_follow, +) + +def _init_synergies(): + """Initialize all synergy definitions.""" + + # Helper to check for specific seq combinations that should NOT get noun+prt synergy + def not_to_before_wa(left_seg_list: Any, right_seg_list: Any) -> bool: + """Return False if left is と (SEQ_TO) and right is は (SEQ_WA).""" + left_seqs = set() + right_seqs = set() + for seg in getattr(left_seg_list, 'segments', []): + info = getattr(seg, 'info', {}) + left_seqs.update(info.get('seq_set', set())) + for seg in getattr(right_seg_list, 'segments', []): + info = getattr(seg, 'info', {}) + right_seqs.update(info.get('seq_set', set())) + # Block synergy if left has と and right has は + if SEQ_TO in left_seqs and SEQ_WA in right_seqs: + return False + return True + + # Create a modified filter_is_noun that also checks the right side + def filter_is_noun_not_to_wa(segment: Any, right_seg_list: Any = None) -> bool: + if not filter_is_noun(segment): + return False + # This will be used via custom synergy below + return True + + # noun + particle (with exclusion for と + は) + def synergy_noun_particle(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Custom noun+particle synergy that excludes と + は.""" + # Check if left is noun + left_nouns = [s for s in seg_list_left.segments if filter_is_noun(s)] + if not left_nouns: + return [] + + # Check if right is particle + particle_filter = filter_in_seq_set(*NOUN_PARTICLES) + right_particles = [s for s in seg_list_right.segments if particle_filter(s)] + if not right_particles: + return [] + + # Check serial + if seg_list_left.end != seg_list_right.start: + return [] + + # Check for と + は case that should be excluded + if not not_to_before_wa(seg_list_left, seg_list_right): + return [] + + # Create synergy + length = seg_list_right.end - seg_list_right.start + score = 10 + 4 * length + + synergy = Synergy( + description="noun+prt", + connector=" ", + score=score, + start=seg_list_left.end, + end=seg_list_right.start, + ) + + return [(seg_list_right, synergy, seg_list_left)] + + register_synergy(synergy_noun_particle) + + # noun + だ + def_generic_synergy( + name="synergy-noun-da", + filter_left=filter_is_noun, + filter_right=filter_in_seq_set(2089020), # だ + description="noun+da", + score=10, + connector=" ", + ) + + # の + だ/です/なんだ + def_generic_synergy( + name="synergy-no-da", + filter_left=filter_in_seq_set(1469800, 2139720), # の, ん + filter_right=filter_in_seq_set(2089020, 1007370, 1928670), # だ, だった, だろう + description="no da/desu", + score=15, + connector=" ", + ) + + # そう + なんだ + def_generic_synergy( + name="synergy-sou-nanda", + filter_left=filter_in_seq_set(2137720), # そう + filter_right=filter_in_seq_set(2140410), # なんだ + description="sou na n da", + score=50, + connector=" ", + ) + + # no-adjective + の + def_generic_synergy( + name="synergy-no-adjectives", + filter_left=filter_is_pos("adj-no"), + filter_right=filter_in_seq_set(1469800), # の + description="no-adjective", + score=15, + connector=" ", + ) + + # na-adjective + な/に + def_generic_synergy( + name="synergy-na-adjectives", + filter_left=filter_is_pos("adj-na"), + filter_right=filter_in_seq_set(2029110, 2028990), # な, に + description="na-adjective", + score=15, + connector=" ", + ) + + # to-adverb + と + def_generic_synergy( + name="synergy-to-adverbs", + filter_left=filter_is_pos("adv-to"), + filter_right=filter_in_seq_set(1008490), # と + description="to-adverb", + score=lambda l, r: 10 + 10 * (l.end - l.start), + connector=" ", + ) + + # noun + 中 + def_generic_synergy( + name="synergy-suffix-chu", + filter_left=filter_is_noun, + filter_right=filter_in_seq_set(1620400, 2083570), # 中 + description="suffix-chu", + score=12, + connector="-", + ) + + # noun + たち + # Use dynamic score based on noun length - longer nouns get higher synergy + # This helps 村人+たち (258+15=273) beat 村+人たち (263) + def_generic_synergy( + name="synergy-suffix-tachi", + filter_left=filter_is_noun, + filter_right=filter_in_seq_set(1416220), # たち + description="suffix-tachi", + score=lambda l, r: 10 + 5 * (l.end - l.start), # +5 per character + connector="-", + ) + + # noun + ぶり + def_generic_synergy( + name="synergy-suffix-buri", + filter_left=filter_is_noun, + filter_right=filter_in_seq_set(1361140), # ぶり + description="suffix-buri", + score=40, + connector="", + ) + + # noun + 性 + def_generic_synergy( + name="synergy-suffix-sei", + filter_left=filter_is_noun, + filter_right=filter_in_seq_set(1375260), # 性 + description="suffix-sei", + score=12, + connector="", + ) + + # お + noun (excluding cases where it splits a valid compound like ごみ) + # Create a combined filter that checks pos AND excludes specific seqs + def filter_o_noun_right(segment: Any) -> bool: + """Filter for nouns that can follow お/ご prefix.""" + # Must be a noun + if not filter_is_pos("n")(segment): + return False + # Exclude みの (straw raincoat) to prevent splitting ごみ + info = getattr(segment, 'info', {}) + seq_set = info.get('seq_set', set()) + excluded_seqs = {1634010, 2845080} # みの seqs + if seq_set.intersection(excluded_seqs): + return False + return True + + def_generic_synergy( + name="synergy-o-prefix", + filter_left=filter_in_seq_set(1270190), # お + filter_right=filter_o_noun_right, + description="o+noun", + score=10, + connector="", + ) + + # 未/不 + noun + def_generic_synergy( + name="synergy-kanji-prefix", + filter_left=filter_in_seq_set(2242840, 1922780, 2423740), # 未, 不 + filter_right=filter_is_pos("n"), + description="kanji prefix+noun", + score=15, + connector="", + ) + + # しちゃ/しては + いけない + def_generic_synergy( + name="synergy-shicha-ikenai", + filter_left=filter_is_compound_end(2028920), # は + filter_right=filter_in_seq_set(1000730, 1612750, 1409110, 2829697, 1587610), + description="shicha ikenai", + score=50, + connector=" ", + ) + + # しか + negative + def synergy_shika_negative(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + start = seg_list_left.end + end = seg_list_right.start + + if start != end: + return [] + + # Filter left for しか + filter_shika = filter_in_seq_set(1005460) + left_segments = [s for s in seg_list_left.segments if filter_shika(s)] + + if not left_segments: + return [] + + # Filter right for negative conjugation + right_segments = [] + for seg in seg_list_right.segments: + info = getattr(seg, 'info', {}) + conj = info.get('conj', []) + for cdata in conj: + if hasattr(cdata, 'prop') and cdata.prop and getattr(cdata.prop, 'neg', False): + right_segments.append(seg) + break + + if not right_segments: + return [] + + syn = Synergy( + description="shika+neg", + connector=" ", + score=50, + start=start, + end=end, + ) + + from himotoki.lookup import SegmentList + new_left = SegmentList( + segments=left_segments, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_segments, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_shika_negative) + + # の + 通り + def_generic_synergy( + name="synergy-no-toori", + filter_left=filter_in_seq_set(1469800), # の + filter_right=filter_in_seq_set(1432920), # 通り + description="no toori", + score=50, + connector=" ", + ) + + # counter + おき + def_generic_synergy( + name="synergy-oki", + filter_left=filter_is_pos("ctr"), + filter_right=filter_in_seq_set(2854117, 2084550), # おき + description="counter+oki", + score=20, + connector="", + ) + + # かどうか + は (whether or not + topic marker) + # This prevents misparses like はま+だ instead of は+まだ + def_generic_synergy( + name="synergy-kadouka-wa", + filter_left=filter_in_seq_set(2087300), # かどうか + filter_right=filter_in_seq_set(2028920), # は + description="kadouka+wa", + score=30, + connector=" ", + ) + + # particle + common adverb synergy + # This boosts patterns like は+まだ, には+まだ, も+まだ etc. + # Common adverbs that follow particles: + # まだ (1527110): still, yet + # もう (1010180): already, soon + # まず (1623080): first of all + # やはり (2084660): as expected + # すでに (1303920): already + # ずっと (1008930): continuously + # もっと (1010210): more + # とても (1008550): very + # かなり (1004920): considerably + # なかなか (1530760): quite, rather + # 少し (1340610): a little + # ちょっと (1008680): a little + # 全然 (1391950): not at all + # 絶対 (1391700): absolutely + # 本当に (1583020): really + # 実は (1311820): actually + # 確かに (1208880): certainly + # 多分 (1397270): probably + # きっと (1399930): surely + # たぶん (1397270): probably + # 結局 (1252670): after all + # 一応 (2423580): tentatively + # とりあえず (1541310): for now + # 相変わらず (1273140): as usual + # いつも (1216780): always + # たまに (1623560): occasionally + # 時々 (1320780): sometimes + # よく (1544660): often + # あまり (1010990): not very + COMMON_ADVERB_SEQS = { + 1527110, # まだ + 1010180, # もう + 1623080, # まず + 2084660, # やはり + 1303920, # すでに (既に) + 1008930, # ずっと + 1010210, # もっと + 1008550, # とても + 1004920, # かなり + 1530760, # なかなか + 1340610, # 少し + 1008680, # ちょっと + 1391950, # 全然 + 1391700, # 絶対 + 1583020, # 本当に + 1311820, # 実は + 1208880, # 確かに + 1397270, # 多分/たぶん + 1399930, # きっと + 1252670, # 結局 + 2423580, # 一応 + 1541310, # とりあえず + 1273140, # 相変わらず + 1216780, # いつも + 1623560, # たまに + 1320780, # 時々 + 1544660, # よく + 1010990, # あまり + } + + # Particles that can precede common adverbs + PARTICLE_SEQS = { + 2028920, # は + 2028930, # が + 2028940, # も + 2028990, # に + 2028980, # で + 2215430, # には + 2028950, # とは + 1007340, # だけ + 1525680, # まで + 1002980, # から + } + + def_generic_synergy( + name="synergy-particle-adverb", + filter_left=filter_in_seq_set(*PARTICLE_SEQS), + filter_right=filter_in_seq_set(*COMMON_ADVERB_SEQS), + description="particle+adverb", + score=20, # Strong synergy to beat noun+copula patterns + connector=" ", + ) + + # === BUG FIX: volitional + とも (1k3) === + # Add synergy for volitional form + とも (seq=1632180) + # This pattern means "even if" (e.g., 反対しようとも = even if they oppose) + # We need to boost とも when it follows a volitional form + def synergy_volitional_tomo(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for volitional form + とも.""" + start = seg_list_left.end + end = seg_list_right.start + + # Must be adjacent + if start != end: + return [] + + # Check if left has volitional conjugation + left_volitional = [] + for seg in seg_list_left.segments: + info = getattr(seg, 'info', {}) + conj = info.get('conj', []) + for cdata in conj: + if hasattr(cdata, 'prop') and cdata.prop: + if getattr(cdata.prop, 'conj_type', None) == 9: # CONJ_VOLITIONAL = 9 + left_volitional.append(seg) + break + + if not left_volitional: + return [] + + # Check if right is とも (seq=1632180) + filter_tomo = filter_in_seq_set(1632180) + right_tomo = [s for s in seg_list_right.segments if filter_tomo(s)] + + if not right_tomo: + return [] + + syn = Synergy( + description="volitional+tomo", + connector=" ", + score=50, # Strong synergy to prefer compound とも + start=start, + end=end, + ) + + from himotoki.lookup import SegmentList + new_left = SegmentList( + segments=left_volitional, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_tomo, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_volitional_tomo) + + # === BUG FIX: verb + なんて (v3f) === + # Add synergy for verb + なんて (seq=1188370) + # This prevents 提出させられるな + ん + て (imperative negative split) + # なんて is an expression meaning "such things as" that follows verb plain form + def synergy_verb_nante(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for verb + なんて.""" + start = seg_list_left.end + end = seg_list_right.start + + # Must be adjacent + if start != end: + return [] + + # Check if left is verb-like (non-negative, non-imperative) + left_verbs = [] + for seg in seg_list_left.segments: + info = getattr(seg, 'info', {}) + posi = info.get('posi', []) + verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', 'v5t', 'v5u', 'vk', 'vs', 'vs-i'} + + if verb_pos.intersection(posi): + # Check it's not imperative negative + conj = info.get('conj', []) + is_imperative_neg = False + for cdata in conj: + if hasattr(cdata, 'prop') and cdata.prop: + if getattr(cdata.prop, 'conj_type', None) == 10: # CONJ_IMPERATIVE + if getattr(cdata.prop, 'neg', False): + is_imperative_neg = True + break + + if not is_imperative_neg: + left_verbs.append(seg) + + if not left_verbs: + return [] + + # Check if right is なんて (seq=1188370) + filter_nante = filter_in_seq_set(1188370) + right_nante = [s for s in seg_list_right.segments if filter_nante(s)] + + if not right_nante: + return [] + + syn = Synergy( + description="verb+nante", + connector=" ", + score=60, # Strong synergy to prefer なんて as expression + start=start, + end=end, + ) + + from himotoki.lookup import SegmentList + new_left = SegmentList( + segments=left_verbs, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_nante, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_verb_nante) + + # === BUG FIX: し (particle) + ただ (adverb) (f5d) === + # Boost し+ただ over した+だ pattern + # し (seq=2086640) is a particle meaning "and/besides" + # ただ (seq=1538900) is an adverb meaning "only/just" + # Pattern: ねーしただ = "isn't, and just..." not "isn't, did, is..." + SEQ_SHI_PARTICLE = 2086640 + SEQ_TADA_ADV = 1538900 + + def synergy_shi_tada(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for し (particle) + ただ (adverb).""" + start = seg_list_left.end + end = seg_list_right.start + + # Must be adjacent + if start != end: + return [] + + # Check if left is し particle (seq=2086640) + filter_shi = filter_in_seq_set(SEQ_SHI_PARTICLE) + left_shi = [s for s in seg_list_left.segments if filter_shi(s)] + + if not left_shi: + return [] + + # Check if right is ただ (seq=1538900) + filter_tada = filter_in_seq_set(SEQ_TADA_ADV) + right_tada = [s for s in seg_list_right.segments if filter_tada(s)] + + if not right_tada: + return [] + + syn = Synergy( + description="shi+tada", + connector=" ", + score=10, # Small boost to prefer し+ただ over した+だ + start=start, + end=end, + ) + + from himotoki.lookup import SegmentList + new_left = SegmentList( + segments=left_shi, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_tada, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_shi_tada) + + # === BUG FIX: verb + よ (particle) (h0z) === + # Boost verb + よ (particle) over adjective stem interpretation + # よ (seq=2029090) is a sentence-ending emphasis particle + # The adjective stem of いい -> よ (euphonic change) causes misidentification + # When よ follows a verb, it's almost always the particle + SEQ_YO_PARTICLE = 2029090 + + def synergy_verb_yo(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for verb + よ (particle).""" + start = seg_list_left.end + end = seg_list_right.start + + # Must be adjacent + if start != end: + return [] + + # Check if left is a verb + left_verbs = [] + verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', 'v5t', 'v5u', 'vk', 'vs', 'vs-i'} + for seg in seg_list_left.segments: + info = getattr(seg, 'info', {}) + posi = info.get('posi', []) + if verb_pos.intersection(posi): + left_verbs.append(seg) + + if not left_verbs: + return [] + + # Check if right is よ particle (seq=2029090) AND is not conjugated + right_yo = [] + for seg in seg_list_right.segments: + info = getattr(seg, 'info', {}) + seq_set = info.get('seq_set', set()) + if SEQ_YO_PARTICLE in seq_set: + # Check that this is NOT a conjugated form + conj = info.get('conj', []) + is_conjugated = False + for cdata in conj: + if hasattr(cdata, 'prop') and cdata.prop: + if getattr(cdata.prop, 'conj_type', None) is not None: + is_conjugated = True + break + if not is_conjugated: + right_yo.append(seg) + + if not right_yo: + return [] + + syn = Synergy( + description="verb+yo", + connector=" ", + score=30, # Strong boost to prefer particle よ after verbs + start=start, + end=end, + ) + + from himotoki.lookup import SegmentList + new_left = SegmentList( + segments=left_verbs, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_yo, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_verb_yo) + + # === BUG FIX: copula + よ (particle) (v42) === + # Boost copula + よ (particle) to prevent よそ mis-segmentation + # When です/だ is followed by よ particle, boost that interpretation + # This prevents "ですよその" from being parsed as "です+よそ+の" + SEQ_DESU = 1628500 # です + SEQ_DA = 2089020 # だ + SEQ_YOSO = 1612400 # よそ (elsewhere) - used for blocking + + def synergy_copula_yo(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for copula + よ (particle) to prevent よそ mis-segmentation.""" + start = seg_list_left.end + end = seg_list_right.start + + # Must be adjacent + if start != end: + return [] + + # Check if left is copula (です or だ) + filter_copula = filter_in_seq_set(SEQ_DESU, SEQ_DA) + left_copula = [s for s in seg_list_left.segments if filter_copula(s)] + if not left_copula: + return [] + + # Check if right is よ particle (seq=SEQ_YO_PARTICLE, not conjugated) + right_yo = [] + for seg in seg_list_right.segments: + info = getattr(seg, 'info', {}) + seq_set = info.get('seq_set', set()) + if SEQ_YO_PARTICLE in seq_set: + # Check that this is NOT a conjugated form + conj = info.get('conj', []) + is_conjugated = False + for cdata in conj: + if hasattr(cdata, 'prop') and cdata.prop: + if getattr(cdata.prop, 'conj_type', None) is not None: + is_conjugated = True + break + if not is_conjugated: + right_yo.append(seg) + + if not right_yo: + return [] + + syn = Synergy( + description="copula+yo", + connector=" ", + score=40, # Strong boost to prefer particle よ after copula + start=start, + end=end, + ) + + from himotoki.lookup import SegmentList + new_left = SegmentList( + segments=left_copula, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_yo, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_copula_yo) + + # 前(まえ) + に synergy - boost noun reading over prefix reading + # When 前 is followed by に particle, prefer まえ (noun) over ぜん (prefix) + def synergy_mae_ni(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for 前(まえ) + に to prefer noun reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for seq=SEQ_MAE_NOUN (まえ reading) + filter_left = filter_in_seq_set(SEQ_MAE_NOUN) + left_mae = [s for s in seg_list_left.segments if filter_left(s)] + if not left_mae: + return [] + + # Filter right for に particle + filter_right = filter_in_seq_set(SEQ_NI) + right_ni = [s for s in seg_list_right.segments if filter_right(s)] + if not right_ni: + return [] + + # Create synergy with high score to push まえ over ぜん + syn = Synergy( + description="mae+ni", + connector=" ", + score=25, # Higher score to ensure まえ wins over ぜん + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_mae, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_ni, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_mae_ni) + + # の + 方(ほう) synergy - boost direction reading over person reading + # When の is followed by 方, prefer ほう (direction) over かた (person) + # Pattern: 〜の方 (towards/that direction) uses ほう reading + def synergy_no_hou(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for の + 方(ほう) to prefer direction reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for の particle + filter_left = filter_in_seq_set(SEQ_NO) + left_no = [s for s in seg_list_left.segments if filter_left(s)] + if not left_no: + return [] + + # Filter right for seq=SEQ_HOU_NOUN (ほう reading) + filter_right = filter_in_seq_set(SEQ_HOU_NOUN) + right_hou = [s for s in seg_list_right.segments if filter_right(s)] + if not right_hou: + return [] + + # Create synergy with high score to push ほう over かた + syn = Synergy( + description="no+hou", + connector=" ", + score=25, # Higher score to ensure ほう wins + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_no, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_hou, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_no_hou) + + # noun + 面(めん) synergy - boost aspect reading over face reading + # When a noun is followed by 面, prefer めん (aspect) over つら (face) + # Pattern: X面 (X aspect/side) like コスト面, 経済面 uses めん reading + def synergy_noun_men(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for noun + 面(めん) to prefer aspect reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for nouns + left_nouns = [s for s in seg_list_left.segments if filter_is_noun(s)] + if not left_nouns: + return [] + + # Filter right for seq=SEQ_MEN_NOUN (めん reading) + filter_right = filter_in_seq_set(SEQ_MEN_NOUN) + right_men = [s for s in seg_list_right.segments if filter_right(s)] + if not right_men: + return [] + + # Create synergy with high score to push めん over つら + syn = Synergy( + description="noun+men", + connector=" ", + score=25, # Higher score to ensure めん wins + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_nouns, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_men, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_noun_men) + + # verb/clause + 人(ひと) synergy - boost person reading over suffix reading + # When 人 follows a verb in plain form or a clause-ending word, + # it should be read as ひと (person), not じん (suffix). + # Pattern: 持つ人, 知らない人, いる人 -- all use ひと + # じん is only used as a suffix after nouns (日本人, アメリカ人) + def synergy_verb_hito(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for verb + 人(ひと) to prefer person reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for verbs or adjectives (clause-ending words) + left_verbs = [] + verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', + 'v5t', 'v5u', 'v5u-s', 'v5k-s', 'v5r-i', 'vk', 'vs', + 'vs-i', 'adj-i', 'adj-ix'} + for seg in seg_list_left.segments: + info = getattr(seg, 'info', {}) + posi = info.get('posi', []) + if verb_pos.intersection(posi): + left_verbs.append(seg) + + if not left_verbs: + return [] + + # Filter right for seq=SEQ_HITO_NOUN (ひと reading) + filter_right = filter_in_seq_set(SEQ_HITO_NOUN) + right_hito = [s for s in seg_list_right.segments if filter_right(s)] + if not right_hito: + return [] + + syn = Synergy( + description="verb+hito", + connector=" ", + score=25, + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_verbs, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_hito, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_verb_hito) + + # 人(ひと) + の synergy - boost person reading when followed by possessive + # When 人 is followed by の (possessive), it should be read as ひと (person). + # Pattern: 人の一生, 人の話, 人の気持ち -- all use ひと + # This complements the verb+hito synergy by handling sentence-initial 人 + SEQ_NO_PARTICLE = 1469800 # の (particle) + + def synergy_hito_no(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for 人(ひと) + の to prefer person reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for seq=SEQ_HITO_NOUN (ひと reading) + filter_left = filter_in_seq_set(SEQ_HITO_NOUN) + left_hito = [s for s in seg_list_left.segments if filter_left(s)] + if not left_hito: + return [] + + # Filter right for の (possessive particle) + filter_right = filter_in_seq_set(SEQ_NO_PARTICLE) + right_no = [s for s in seg_list_right.segments if filter_right(s)] + if not right_no: + return [] + + syn = Synergy( + description="hito+no", + connector=" ", + score=30, # Strong enough to override じん + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_hito, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_no, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_hito_no) + + # する + 中(なか) synergy - boost middle reading over suffix reading + # When 中 follows a verb (する中, 激化する中, etc.), it means "amid/while" + # and should be read as なか, not ちゅう (which is a suffix like 勉強中) + def synergy_verb_naka(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for verb + 中(なか) to prefer 'middle/amid' reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for verbs in plain form (non-past, non-negative) + left_verbs = [] + verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', + 'v5t', 'v5u', 'v5u-s', 'v5k-s', 'v5r-i', 'vk', 'vs', + 'vs-i'} + for seg in seg_list_left.segments: + info = getattr(seg, 'info', {}) + posi = info.get('posi', []) + if verb_pos.intersection(posi): + left_verbs.append(seg) + + if not left_verbs: + return [] + + # Filter right for seq=SEQ_NAKA_NOUN (なか reading) + filter_right = filter_in_seq_set(SEQ_NAKA_NOUN) + right_naka = [s for s in seg_list_right.segments if filter_right(s)] + if not right_naka: + return [] + + syn = Synergy( + description="verb+naka", + connector=" ", + score=25, + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_verbs, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_naka, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_verb_naka) + + # が + 止まる(とまる) synergy - prefer "to stop" over "to stay/remain" + # When 止まる follows が (subject marker), it usually means "stops" (とまる). + # "とどまる" (to stay/remain) typically takes に (にとどまる). + # Pattern: 時間が止まった, 車が止まった, 心臓が止まった + def synergy_ga_tomaru(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for が + 止まる(とまる) to prefer 'to stop' reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for が particle + filter_ga = filter_in_seq_set(SEQ_GA) + left_ga = [s for s in seg_list_left.segments if filter_ga(s)] + if not left_ga: + return [] + + # Filter right for とまる (seq=1310620, to stop) + filter_tomaru = filter_in_seq_set(SEQ_TOMARU) + right_tomaru = [s for s in seg_list_right.segments if filter_tomaru(s)] + if not right_tomaru: + return [] + + syn = Synergy( + description="ga+tomaru", + connector=" ", + score=25, + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_ga, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_tomaru, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_ga_tomaru) + + # は + 辛い(つらい) synergy - prefer "painful/hard" over "spicy" + # When 辛い follows は (topic marker), it usually means "hard/painful" (つらい). + # "からい" (spicy/hot) is primarily used in food contexts with direct modification. + # Pattern: リハビリは辛い, 別れは辛い, 現実は辛い + def synergy_wa_tsurai(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: + """Synergy for は + 辛い(つらい) to prefer 'painful/hard' reading.""" + from himotoki.lookup import SegmentList + + # Check serial - must be adjacent + if seg_list_left.end != seg_list_right.start: + return [] + + # Filter left for は particle + filter_wa = filter_in_seq_set(SEQ_WA) + left_wa = [s for s in seg_list_left.segments if filter_wa(s)] + if not left_wa: + return [] + + # Filter right for つらい (seq=1365860, painful/hard) + filter_tsurai = filter_in_seq_set(SEQ_TSURAI) + right_tsurai = [s for s in seg_list_right.segments if filter_tsurai(s)] + if not right_tsurai: + return [] + + syn = Synergy( + description="wa+tsurai", + connector=" ", + score=50, + start=seg_list_left.end, + end=seg_list_right.start, + ) + + new_left = SegmentList( + segments=left_wa, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + new_right = SegmentList( + segments=right_tsurai, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + + return [(new_right, syn, new_left)] + + register_synergy(synergy_wa_tsurai) + + + +# Initialize synergies on module load +def _init_penalties(): + """Initialize all penalty definitions.""" + + # と + は should be penalized to prefer compound とは (seq=2028950) + # The noun+prt synergy gives +14 to と+は which makes it beat とは (24 vs 22) + # This penalty counteracts that so とは wins + # NOTE: This must come BEFORE other penalties so it applies first + def has_seq_simple(seqs: set): + """Check if segment_list has any of the given seq numbers.""" + def _filter(segment_list: Any) -> bool: + segments = getattr(segment_list, 'segments', []) + for seg in segments: + info = getattr(seg, 'info', {}) + seq_set = info.get('seq_set', set()) + if seqs.intersection(seq_set): + return True + return False + return _filter + + def_generic_penalty( + name="penalty-to-wa", + test_left=has_seq_simple({1008490}), # と + test_right=has_seq_simple({2028920}), # は + description="to+wa-penalty", + score=-20, + serial=True, + ) + + # に + つれ should be penalized to prefer compound につれ (seq=2136050) + # につれ has score 36, but に(11)+つれ(40)=51 wins otherwise + # Need penalty of at least -(51-36)=-15, but make it larger to be safe + # つれ seqs include: 10351890, 1434020, 10097136, 1559290, 1434120, 10351981 + def_generic_penalty( + name="penalty-ni-tsure", + test_left=has_seq_simple({2028990}), # に + test_right=has_seq_simple({10351890, 1434020, 10097136, 1559290, 1434120, 10351981}), # つれ + description="ni+tsure-penalty", + score=-30, + serial=True, + ) + + # お + すすめ should be penalized to prefer compound おすすめ (seq=1002150) + # おすすめ has score 64, but お(10)+すすめ(90)=100 wins otherwise + # Need penalty of at least -36 to make them equal + # お seqs: 1343610 (common=12), 1485770 (common=5), 2089690, 2268350, 2603520, 2742870, 2826528 + # すすめ seqs: 1210900 (noun), 10074000 (verb conjugation), 1365980, 1365990 + def_generic_penalty( + name="penalty-o-susume", + test_left=has_seq_simple({1343610, 1485770, 2089690, 2268350, 2603520, 2742870, 2826528}), # お + test_right=has_seq_simple({1210900, 10074000, 1365980, 1365990}), # すすめ + description="o+susume-penalty", + score=-40, + serial=True, + ) + + # 人がい + たら should be penalized to prefer 人 + が + いたら + # 人がい is a conjugation of 人がいい (seq=10043332, 2250200) + # 人がい(120)+たら(68)=188 beats 人(16)+が(11)+いたら(90)=117 + # Need penalty of at least -71 to make them equal + # たら seqs: 1408160, 1416790, 2029050, 11435516, 11679733 + def_generic_penalty( + name="penalty-hitogai-tara", + test_left=has_seq_simple({10043332, 2250200}), # 人がい / 人がいい + test_right=has_seq_simple({1408160, 1416790, 2029050, 11435516, 11679733}), # たら + description="hitogai+tara-penalty", + score=-75, + serial=True, + ) + + # ご + みの should be penalized to prefer compound ごみ (seq=1369900) + # ごみ + の = 12 + 16 = 28 + # ご(10) + みの(12) = 22, plus o+noun synergy(+10) = 32 + # Need penalty of at least -6 to make ごみ + の win (28 > 26) + # Using -15 to ensure ごみ is strongly preferred + # ご (honorific prefix): seq=1270190 + # みの: seq=1634010 (straw raincoat), 2845080 (Mino province) + def_generic_penalty( + name="penalty-go-mino", + test_left=has_seq_simple({1270190}), # ご (honorific prefix) + test_right=has_seq_simple({1634010, 2845080}), # みの + description="go+mino-penalty", + score=-15, + serial=True, + ) + + # わかん + ない should be penalized to prefer compound わかんない (seq=2158960) + # わかんない has score 125, but わかん(110)+ない(40)=150 wins otherwise + # Need penalty of at least -(150-125)=-25 to make わかんない win + # Using -30 to ensure it's strongly preferred + # わかん seqs: 10256789, 10256836 (conjugated forms of 分かる) + # ない seqs: 10452328, 1529520 (negative adjective/suffix) + def_generic_penalty( + name="penalty-wakan-nai", + test_left=has_seq_simple({10256789, 10256836}), # わかん + test_right=has_seq_simple({10452328, 1529520}), # ない + description="wakan+nai-penalty", + score=-30, + serial=True, + ) + + # 知らん + けど should be penalized to prefer compound 知らんけど (seq=2856919) + # 知らんけど exists as a single expression (Kansai dialect "I dunno but") + # 知らん seqs include various conjugated forms + # けど seqs: 1004200 + def_generic_penalty( + name="penalty-shiran-kedo", + test_left=has_seq_simple({10350776}), # 知らん (conjugated 知る) + test_right=has_seq_simple({1004200}), # けど + description="shiran+kedo-penalty", + score=-100, # Strong penalty since 知らんけど is a specific expression + serial=True, + ) + + # はま + だ should be penalized to prefer は + まだ + # はま (seq=1490710, beach) + だ (copula) incorrectly beats は + まだ (still) + # はま+だ gets 66 (with noun+だ synergy), は+まだ gets 51 + # With particle+adverb synergy (+20), は+まだ gets 71 + # But we still need a penalty to be safe: -20 makes はま+だ = 46, safely below 71 + def_generic_penalty( + name="penalty-hama-da", + test_left=has_seq_simple({1490710, 2084350}), # はま (beach, other) + test_right=has_seq_simple({2089020}), # だ (copula) + description="hama+da-penalty", + score=-20, + serial=True, + ) + + # 人たち after single-kanji word should be penalized when compound word exists + # 村 + 人たち (263) vs 村人 + たち (258) + # This penalty makes 村+人たち = 263-10 = 253, so 村人+たち (258) wins + def penalty_hitotachi_split(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: + """Penalize single-kanji + 人たち to prefer compound + たち.""" + start = seg_list_left.end + end = seg_list_right.start + + # Must be serial + if start != end: + return None + + # Check if right is 人たち (seq=1368740) + segments_right = getattr(seg_list_right, 'segments', []) + has_hitotachi = False + for seg in segments_right: + info = getattr(seg, 'info', {}) + seq_set = info.get('seq_set', set()) + if 1368740 in seq_set: + has_hitotachi = True + break + + if not has_hitotachi: + return None + + # Check if left is a single-character word (likely kanji that could combine) + segments_left = getattr(seg_list_left, 'segments', []) + for seg in segments_left: + word = getattr(seg, 'word', None) + if word and hasattr(word, 'text'): + text = word.text + if len(text) == 1: + # Single character - check if it's kanji + from himotoki.characters import is_kanji + if is_kanji(text): + return Synergy( + description="single-kanji+hitotachi-penalty", + connector=" ", + score=-15, # Penalty to prefer compound+たち + start=start, + end=end, + ) + + return None + + register_penalty(penalty_hitotachi_split) + + # === BUG FIX: につけ (7g2) === + # に + つけ should be penalized to prefer compound につけ (seq=2840365) + # につけ is a grammatical expression meaning "whenever" + # つけ seqs: 1495750 (noun), conjugations of つける (10092135, 10092153, etc.) + def_generic_penalty( + name="penalty-ni-tsuke", + test_left=has_seq_simple({2028990}), # に + test_right=has_seq_simple({1495750, 10092135, 10092153, 10092153, 1495740}), # つけ + description="ni+tsuke-penalty", + score=-30, + serial=True, + ) + + # === BUG FIX: 未だに (bzl) === + # 未だ + に should be penalized to prefer compound 未だに (seq=1527140) + # 未だに is an adverb meaning "still" or "even now" + # 未だ seqs: 1527110 + def_generic_penalty( + name="penalty-mada-ni", + test_left=has_seq_simple({1527110}), # 未だ + test_right=has_seq_simple({2028990}), # に + description="mada+ni-penalty", + score=-30, + serial=True, + ) + + # === BUG FIX: と + も after volitional (1k3) === + # と + も should be penalized to prefer compound とも (seq=1632180) + # when following a volitional form + # とも is a particle meaning "even if" after volitional form + def_generic_penalty( + name="penalty-to-mo", + test_left=has_seq_simple({1008490}), # と + test_right=has_seq_simple({2028940}), # も + description="to+mo-penalty", + score=-30, + serial=True, + ) + + # === BUG FIX: たん boundary (klu) === + # verb stem + たん should be penalized when past+ん is intended + # たん (seq=2646370) is a prefix/noun, not past+explanatory + # This prevents 仕掛け + たん instead of 仕掛けた + ん + # We need to check if left is a verb stem (continuative form) + # and right is たん (which should be た + ん instead) + def penalty_verb_tan(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: + """Penalize verb stem + たん when past+ん is intended.""" + start = seg_list_left.end + end = seg_list_right.start + + # Must be serial + if start != end: + return None + + # Check if right is たん (seq=2646370) + segments_right = getattr(seg_list_right, 'segments', []) + has_tan = False + for seg in segments_right: + info = getattr(seg, 'info', {}) + seq_set = info.get('seq_set', set()) + if 2646370 in seq_set: + has_tan = True + break + + if not has_tan: + return None + + # Check if left ends with a word that could be a verb stem (masu-stem) + # Words like 仕掛け (noun) can also be verb stems + # Penalty should be applied regardless to prefer past+explanatory + segments_left = getattr(seg_list_left, 'segments', []) + for seg in segments_left: + info = getattr(seg, 'info', {}) + posi = info.get('posi', []) + # Check if it's a noun that could be a verb stem (many verb stems are also nouns) + # or a verb form + verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', 'v5t', 'v5u', 'vk', 'vs', 'vs-i', 'n'} + if verb_pos.intersection(posi): + return Synergy( + description="verb-stem+tan-penalty", + connector=" ", + score=-400, # Very strong penalty - diff is often 300+ points + start=start, + end=end, + ) + + return None + + register_penalty(penalty_verb_tan) + + # Short kana words together + def_generic_penalty( + name="penalty-short", + test_left=filter_short_kana(1), + test_right=filter_short_kana(1, except_list=['と']), + description="short", + score=-9, + serial=False, + ) + + # Semi-final particle not at end + def penalty_semi_final(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: + from himotoki.lookup import SEMI_FINAL_PRT + + # Check if left has semi-final particle + filter_fn = filter_in_seq_set(*SEMI_FINAL_PRT) + has_semi_final = any(filter_fn(s) for s in seg_list_left.segments) + + if not has_semi_final: + return None + + return Synergy( + description="semi-final not final", + connector=" ", + score=-15, + start=seg_list_left.end, + end=seg_list_right.start, + ) + + register_penalty(penalty_semi_final) + + +def _init_segfilters(): + """Initialize all segfilter definitions.""" + + # Auxiliary verbs must follow continuative form + AUX_VERBS = {1342560} # 初める/そめる + def_segfilter_must_follow( + name="segfilter-aux-verb", + filter_left=filter_is_conjugation(13), # Continuative + filter_right=filter_in_seq_set(*AUX_VERBS), + ) + + # いる must not follow 終わる (つ + いる conflict) + def_segfilter_must_follow( + name="segfilter-tsu-iru", + filter_left=lambda s: not filter_in_seq_set(2221640)(s), + filter_right=filter_in_seq_set(1577980), # いる + allow_first=True, + ) + + # ん/んだ must not follow simple particles + def_segfilter_must_follow( + name="segfilter-n", + filter_left=lambda s: not filter_in_seq_set_simple(*NOUN_PARTICLES)(s), + filter_right=filter_in_seq_set(2139720, 2849370, 2849387), # ん, んだ + allow_first=True, + ) + + # を + 枯らす + def_segfilter_must_follow( + name="segfilter-wokarasu", + filter_left=filter_in_seq_set(2029010), # を + filter_right=filter_in_seq_set(2087020), + ) + + # Bad endings + def_segfilter_must_follow( + name="segfilter-badend", + filter_left=lambda s: False, + filter_right=filter_is_compound_end_text("ちゃい", "いか", "とか", "とき", "い"), + ) + + # じゃない must not follow は compound + def_segfilter_must_follow( + name="segfilter-janai", + filter_left=lambda s: not filter_is_compound_end(2028920)(s), + filter_right=filter_in_seq_set(1529520, 1296400, 2139720), # ない, ある, ん + allow_first=True, + ) + + # だ + する (dashi problem) + def segfilter_dashi_fn(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + from himotoki.lookup import SegmentList + + # Right must be する/して/し + filter_right = filter_in_seq_set(1157170, 2424740, 1305070) + satisfies_right = [s for s in seg_list_right.segments if filter_right(s)] + contradicts_right = [s for s in seg_list_right.segments if not filter_right(s)] + + if not satisfies_right: + return [(seg_list_left, seg_list_right)] + + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Left must not be だ without で + def left_ok(s): + info = getattr(s, 'info', {}) + seq_set = info.get('seq_set', []) + if 2089020 not in seq_set: # だ + return True + if 2028980 in seq_set: # で + return True + return False + + satisfies_left = [s for s in seg_list_left.segments if left_ok(s)] + + if satisfies_left: + return [(seg_list_left, seg_list_right)] + + if contradicts_right: + return [( + seg_list_left, + SegmentList( + segments=contradicts_right, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )] + + return [] + + register_segfilter(segfilter_dashi_fn) + + # Honorifics must follow noun-like words + HONORIFICS = {1247260} # 君 + def_segfilter_must_follow( + name="segfilter-honorific", + filter_left=lambda s: not filter_in_seq_set(*NOUN_PARTICLES)(s), + filter_right=filter_in_seq_set(*HONORIFICS), + ) + + # 君/くん (suffix) must NOT be followed by particles + # When 君 is followed by a particle (と, は, が, etc.), it's the pronoun きみ, + # not the suffix くん. Remove くん from candidates when followed by particles. + from himotoki.constants import SEQ_KUN, NOUN_PARTICLES as NP + def segfilter_kun_before_particle(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Remove くん (suffix) when followed by a particle.""" + from himotoki.lookup import SegmentList + + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + # Check if right is a particle + filter_particle = filter_in_seq_set(*NP) + right_is_particle = any(filter_particle(s) for s in seg_list_right.segments) + + if not right_is_particle: + return [(seg_list_left, seg_list_right)] + + # Right is a particle - remove くん from left + filter_kun = filter_in_seq_set(SEQ_KUN) + left_without_kun = [s for s in seg_list_left.segments if not filter_kun(s)] + + if not left_without_kun: + # All segments were くん - this path is blocked + return [] + + new_left = SegmentList( + segments=left_without_kun, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ) + return [(new_left, seg_list_right)] + + register_segfilter(segfilter_kun_before_particle) + + # === BUG FIX: Block patterns where compound expressions exist === + + # Segfilter: Block に + つけ to prefer につけ (7g2) + # につけ (seq=2840365) is a grammatical expression "whenever" + def segfilter_ni_tsuke(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Block に + つけ pattern to prefer につけ as compound.""" + from himotoki.lookup import SegmentList + + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + # Check if left is に (seq=2028990) + filter_ni = filter_in_seq_set(2028990) + left_ni = [s for s in seg_list_left.segments if filter_ni(s)] + left_other = [s for s in seg_list_left.segments if not filter_ni(s)] + + # Check if right is つけ (seq=1495750 or conjugations) + filter_tsuke = filter_in_seq_set(1495750, 10092135, 10092153, 1495740) + right_tsuke = [s for s in seg_list_right.segments if filter_tsuke(s)] + right_other = [s for s in seg_list_right.segments if not filter_tsuke(s)] + + # If not the pattern we're blocking, pass through + if not left_ni or not right_tsuke: + return [(seg_list_left, seg_list_right)] + + results = [] + # Allow left_other with any right + if left_other: + results.append(( + SegmentList( + segments=left_other, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ), + seg_list_right, + )) + + # Allow left_ni with right_other (non-tsuke) + if right_other: + results.append(( + seg_list_left, + SegmentList( + segments=right_other, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )) + + return results if results else [] + + register_segfilter(segfilter_ni_tsuke) + + # Segfilter: Block 未だ + に to prefer 未だに (bzl) + # 未だに (seq=1527140) is an adverb "still/even now" + def segfilter_mada_ni(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Block 未だ + に pattern to prefer 未だに as compound.""" + from himotoki.lookup import SegmentList + + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + # Check if left is 未だ (seq=1527110) + filter_mada = filter_in_seq_set(1527110) + left_mada = [s for s in seg_list_left.segments if filter_mada(s)] + left_other = [s for s in seg_list_left.segments if not filter_mada(s)] + + # Check if right is に (seq=2028990) + filter_ni = filter_in_seq_set(2028990) + right_ni = [s for s in seg_list_right.segments if filter_ni(s)] + right_other = [s for s in seg_list_right.segments if not filter_ni(s)] + + # If not the pattern we're blocking, pass through + if not left_mada or not right_ni: + return [(seg_list_left, seg_list_right)] + + results = [] + # Allow left_other with any right + if left_other: + results.append(( + SegmentList( + segments=left_other, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ), + seg_list_right, + )) + + # Allow left_mada with right_other (non-ni) + if right_other: + results.append(( + seg_list_left, + SegmentList( + segments=right_other, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )) + + return results if results else [] + + register_segfilter(segfilter_mada_ni) + + # Segfilter: Block と + も to prefer とも (1k3) + # とも (seq=1632180) is a particle meaning "even if" + def segfilter_to_mo(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Block と + も pattern to prefer とも as compound particle.""" + from himotoki.lookup import SegmentList + + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + # Check if left is と (seq=1008490) + filter_to = filter_in_seq_set(1008490) + left_to = [s for s in seg_list_left.segments if filter_to(s)] + left_other = [s for s in seg_list_left.segments if not filter_to(s)] + + # Check if right is も (seq=2028940) + filter_mo = filter_in_seq_set(2028940) + right_mo = [s for s in seg_list_right.segments if filter_mo(s)] + right_other = [s for s in seg_list_right.segments if not filter_mo(s)] + + # If not the pattern we're blocking, pass through + if not left_to or not right_mo: + return [(seg_list_left, seg_list_right)] + + results = [] + # Allow left_other with any right + if left_other: + results.append(( + SegmentList( + segments=left_other, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ), + seg_list_right, + )) + + # Allow left_to with right_other (non-mo) + if right_other: + results.append(( + seg_list_left, + SegmentList( + segments=right_other, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )) + + return results if results else [] + + register_segfilter(segfilter_to_mo) + + # === BUG FIX: Block verb-negative-imperative + ん (nwd, 5zp) === + # Pattern: verb ending with negative imperative な + ん should be blocked + # This forces the segmenter to use verb + なんて instead of verbな + ん + て + # Example: 発動させるな + ん + て → 発動させる + なんて + SEQ_N_PARTICLE = 2139720 # ん (particle/interjection) + + def segfilter_neg_imperative_n(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Block verb-negative-imperative + ん pattern to prefer verb + なんて.""" + from himotoki.lookup import SegmentList + + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + # Check if left has any negative imperative verbs + left_neg_imp = [] + left_other = [] + for seg in seg_list_left.segments: + info = getattr(seg, 'info', {}) + conj = info.get('conj', []) + is_neg_imperative = False + for cdata in conj: + if hasattr(cdata, 'prop') and cdata.prop: + # Check for imperative + negative + if getattr(cdata.prop, 'conj_type', None) == 10: # CONJ_IMPERATIVE + if getattr(cdata.prop, 'neg', False): + is_neg_imperative = True + break + + if is_neg_imperative: + left_neg_imp.append(seg) + else: + left_other.append(seg) + + # Check if right is ん (seq=2139720) + filter_n = filter_in_seq_set(SEQ_N_PARTICLE) + right_n = [s for s in seg_list_right.segments if filter_n(s)] + right_other = [s for s in seg_list_right.segments if not filter_n(s)] + + # If not the pattern we're blocking, pass through + if not left_neg_imp or not right_n: + return [(seg_list_left, seg_list_right)] + + results = [] + # Allow left_other with any right + if left_other: + results.append(( + SegmentList( + segments=left_other, + start=seg_list_left.start, + end=seg_list_left.end, + matches=seg_list_left.matches, + ), + seg_list_right, + )) + + # Allow left_neg_imp with right_other (non-ん) + if right_other: + results.append(( + seg_list_left, + SegmentList( + segments=right_other, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )) + + return results if results else [] + + register_segfilter(segfilter_neg_imperative_n) + + # === Segfilter: Remove いくさ reading after nouns === + # 戦(いくさ, seq=1587140) is an archaic standalone word for "war". + # When 戦 follows a noun, it should be read as せん (match/battle) + # as part of a compound (e.g., タイトル戦). Since せん gets culled + # by IDENTICAL_WORD_SCORE_CUTOFF (score 5 vs いくさ score 16), + # removing いくさ from the split path lets the compound path win. + from himotoki.constants import SEQ_IKUSA_NOUN + def segfilter_noun_ikusa(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Remove いくさ reading of 戦 when preceded by a noun.""" + from himotoki.lookup import SegmentList + + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + # Check if left has a noun + left_has_noun = any(filter_is_noun(s) for s in seg_list_left.segments) + if not left_has_noun: + return [(seg_list_left, seg_list_right)] + + # Remove いくさ from right segments + filter_ikusa = filter_in_seq_set(SEQ_IKUSA_NOUN) + right_without_ikusa = [s for s in seg_list_right.segments if not filter_ikusa(s)] + + if not right_without_ikusa: + # All segments were いくさ - block this split so compound wins + return [] + + if len(right_without_ikusa) == len(seg_list_right.segments): + # No いくさ found, pass through unchanged + return [(seg_list_left, seg_list_right)] + + new_right = SegmentList( + segments=right_without_ikusa, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ) + return [(seg_list_left, new_right)] + + register_segfilter(segfilter_noun_ikusa) + + # === Segfilter: Block ないよう (内容/内用/内洋) when matched from kana === + # When ないよう appears in hiragana, it's almost always ない+よう (negative purposive) + # rather than the kanji words 内容/内用/内洋. Only allow these seqs when matched from kanji. + SEQ_NAIYOU_KANA = {1459400, 1459440, 2862582} # All words with reading ないよう + def segfilter_naiyou_kana(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Filter out 内容/内用/内洋 when matched from kana (ないよう) instead of kanji.""" + from himotoki.lookup import SegmentList + + def is_naiyou_from_kana(seg): + if not hasattr(seg, 'word'): + return False + if seg.word.seq not in SEQ_NAIYOU_KANA: + return False + # Check if this came from kana matching + return seg.word.word_type == 'kana' + + filtered = [s for s in seg_list_right.segments if not is_naiyou_from_kana(s)] + + if len(filtered) == len(seg_list_right.segments): + # Nothing filtered, pass through + return [(seg_list_left, seg_list_right)] + + if not filtered: + # All filtered out - return empty result so this path is blocked + # The alternative path (ない + よう) should be taken instead + return [] + + return [( + seg_list_left, + SegmentList( + segments=filtered, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )] + + register_segfilter(segfilter_naiyou_kana) + + # === Segfilter: Block ところが (conjunction) after rentaikei forms === + # When ところが follows a noun-modifying verb form (rentaikei), it should be + # split as ところ (place) + が (particle), not the conjunction "however". + # Rentaikei forms: Non-past (type 1), Past (type 2), or any verb/adj that can modify nouns + # Example: 分からないところがある → 分からない + ところ + が + ある + SEQ_TOKOROGA = 1008570 # ところが (conjunction) + RENTAIKEI_CONJ_TYPES = {1, 2} # Non-past, Past - both can modify nouns + + def segfilter_tokoroga(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Filter out ところが (conjunction) when preceded by a noun-modifying form.""" + from himotoki.lookup import SegmentList + + # If no left context, allow ところが (sentence-initial is valid as conjunction) + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + def is_tokoroga_conj(seg): + return seg.word.seq == SEQ_TOKOROGA + + def is_rentaikei(seg): + """Check if segment ends with a noun-modifying (rentaikei) form.""" + conj_list = seg.info.get('conj', []) + if not conj_list: + # No conjugation - check if it's a noun or noun-like + posi = seg.info.get('posi', []) + if any(p in posi for p in ['n', 'adj-i', 'adj-na']): + return True + return False + # Check if any conjugation is rentaikei (can modify nouns) + for conj_data in conj_list: + if hasattr(conj_data, 'prop') and conj_data.prop: + if conj_data.prop.conj_type in RENTAIKEI_CONJ_TYPES: + return True + return False + + # Filter segments + tokoroga = [s for s in seg_list_right.segments if is_tokoroga_conj(s)] + other = [s for s in seg_list_right.segments if not is_tokoroga_conj(s)] + + # If no ところが, pass through + if not tokoroga: + return [(seg_list_left, seg_list_right)] + + # Check if left has rentaikei forms + rentaikei_left = [s for s in seg_list_left.segments if is_rentaikei(s)] + + # If left has rentaikei, block ところが (conjunction) + if rentaikei_left: + if other: + return [( + seg_list_left, + SegmentList( + segments=other, + start=seg_list_right.start, + end=seg_list_right.end, + matches=seg_list_right.matches, + ), + )] + # No other options - block this path entirely + return [] + + # No rentaikei on left, allow ところが + return [(seg_list_left, seg_list_right)] + + register_segfilter(segfilter_tokoroga) + + # === Segfilter: Block たん/たんだ after masu-stem verb forms === + # たん/たんだ as standalone words should not follow + # a masu-stem (continuative form) or noun that could be a verb stem. + # It should be た + ん (+ だ) instead. + # Example: 仕掛けたんだ → 仕掛けた + ん + だ, NOT 仕掛け + たん + だ + + def segfilter_tan_after_stem(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: + """Block たん/たんだ when preceded by continuative/masu-stem form.""" + from himotoki.lookup import SegmentList + + # If no left context, allow たん at sentence start + if seg_list_left is None: + return [(seg_list_left, seg_list_right)] + + # Must be adjacent + if seg_list_left.end != seg_list_right.start: + return [(seg_list_left, seg_list_right)] + + def is_tan_tanda(seg): + """Check if segment text is たん or たんだ.""" + return seg.word.text in ('たん', 'たんだ') + + def is_masu_stem_or_verb_noun(seg): + """Check if segment is a masu-stem form or a noun that could be a verb stem.""" + conj_list = seg.info.get('conj', []) + if conj_list: + for conj_data in conj_list: + if hasattr(conj_data, 'prop') and conj_data.prop: + # Conj type 13 = Continuative (~masu stem) + if conj_data.prop.conj_type == 13: + return True + # Also check if it's a noun that matches a verb's masu-stem pattern + # Many nouns like 仕掛け are also verb stems + posi = seg.info.get('posi', []) + if 'n' in posi: + # Check if the word looks like a potential verb stem + # (ends with け/き/し/り/ち/み/び/ぎ/え/い/etc.) + text = seg.word.text + if text and any(text.endswith(k) for k in 'けきしりちみびぎにえいてれねめべげ'): + return True + return False + + # Filter segments + tan_segs = [s for s in seg_list_right.segments if is_tan_tanda(s)] + other_segs = [s for s in seg_list_right.segments if not is_tan_tanda(s)] + + # If no たん/たんだ, pass through + if not tan_segs: + return [(seg_list_left, seg_list_right)] + + # Check if left has masu-stem or verb-like noun + has_stem = any(is_masu_stem_or_verb_noun(s) for s in seg_list_left.segments) + + if has_stem: + # Block たん/たんだ when preceded by masu-stem/verb-noun + if other_segs: + # Return only non-たん segments + new_right = SegmentList( + start=seg_list_right.start, + end=seg_list_right.end, + segments=other_segs + ) + return [(seg_list_left, new_right)] + else: + # No alternatives - block this path + return [] + + # Not a stem, allow たん/たんだ + return [(seg_list_left, seg_list_right)] + + register_segfilter(segfilter_tan_after_stem) + + +_init_synergies() +_init_penalties() +_init_segfilters() diff --git a/himotoki/loading/jmdict.py b/himotoki/loading/jmdict.py index 94c24df..5d2ec63 100644 --- a/himotoki/loading/jmdict.py +++ b/himotoki/loading/jmdict.py @@ -318,11 +318,8 @@ def load_entry( session.delete(existing) session.flush() - # Serialize original content for storage - content = etree.tostring(entry_elem, encoding='unicode') - - # Create entry record - entry = Entry(seq=seq, content=content, root_p=True) + # Create entry record (content column kept for schema compatibility, not populated) + entry = Entry(seq=seq, content="", root_p=True) session.add(entry) # Parse kanji readings (k_ele elements) @@ -453,6 +450,14 @@ def load_jmdict( with get_session() as session: add_errata(session) + # Optimize database after build + from himotoki.db.connection import analyze, vacuum + logger.info("Running ANALYZE...") + analyze() + logger.info("Running VACUUM (this may take several minutes)...") + vacuum() + logger.info("Database optimization complete") + return count diff --git a/himotoki/lookup.py b/himotoki/lookup.py deleted file mode 100644 index 15e750d..0000000 --- a/himotoki/lookup.py +++ /dev/null @@ -1,2179 +0,0 @@ -""" -Word lookup and scoring module for himotoki. -Ports ichiran's dict.lisp word lookup and scoring functionality. - -This module provides: -- find_word: Database lookup for words -- calc_score: Complex scoring algorithm for word segments -- Segment/SegmentList: Data structures for word matches -- Length multipliers and scoring coefficients -""" - -from dataclasses import dataclass, field -from typing import Optional, List, Dict, Tuple, Union, Any, Set -from functools import lru_cache -from collections import OrderedDict - -from himotoki.raw_types import RawKanaReading, RawKanjiReading - -from sqlalchemy import select, and_, or_, func -from sqlalchemy.orm import Session - -from himotoki.db.models import ( - Entry, KanjiText, KanaText, Sense, Gloss, SenseProp, - Conjugation, ConjProp, ConjSourceReading, -) -from himotoki.characters import ( - is_kana, is_katakana, is_hiragana, has_kanji, as_hiragana, - mora_length, count_char_class, get_char_class, -) - -# Import CounterText for type checking in calc_score -# This is imported here to avoid circular imports -def _is_counter_text(obj) -> bool: - """Check if object is a CounterText without importing the class.""" - return type(obj).__name__ == 'CounterText' - - -# ============================================================================ -# Constants (ported from ichiran's dict-errata.lisp) -# ============================================================================ - -# Maximum word length to search for -MAX_WORD_LENGTH = 50 - -# Score cutoff - words below this score are filtered out -# This must filter out ONLY bad kana spellings, and NOT filter out any kanji spellings -SCORE_CUTOFF = 5 - -# Length coefficient sequences (from ichiran's *length-coeff-sequences*) -# Format: coefficient for mora length 1, 2, 3, 4, 5... -LENGTH_COEFF_SEQUENCES = { - 'strong': [0, 1, 8, 24, 40, 60], # Index 0 unused, 1-based - 'weak': [0, 1, 4, 9, 16, 25, 36], - 'tail': [0, 4, 9, 16, 24], - 'ltail': [0, 4, 12, 18, 24], -} - -# Cutoff ratio for identical word scores -IDENTICAL_WORD_SCORE_CUTOFF = 0.5 - -# Gap penalty per character for ungapped text -GAP_PENALTY = -500 - -# Parts of speech that can be conjugated -POS_WITH_CONJ_RULES = frozenset([ - 'v1', 'v1s', 'v5aru', 'v5b', 'v5g', 'v5k', 'v5k-s', 'v5m', 'v5n', - 'v5r', 'v5r-i', 'v5s', 'v5t', 'v5u', 'v5u-s', 'v5uru', 'vk', 'vs', - 'vs-i', 'vs-s', 'vz', 'adj-i', 'adj-ix', 'adj-na', 'adj-no', -]) - -# Copulae seqs (だ, です, etc.) - from ichiran's *copulae* -COPULAE: Set[int] = {2089020, 1628500} # だ, です - -# Skip words - seq of words that aren't really words, like suffixes etc. -# From ichiran's *skip-words* -SKIP_WORDS: Set[int] = { - 2458040, # てもいい - 2822120, # ても良い - 2013800, # ちゃう - 2108590, # とく - 2029040, # ば - 2428180, # い - 2654250, # た - 2561100, # うまいな - 2210270, # ませんか - 2210710, # ましょうか - 2257550, # ない - 2210320, # ません - 2017560, # たい - 2394890, # とる - 2194000, # であ - 2568000, # れる/られる - 2537250, # しようとする - 2760890, # 三箱 - 2831062, # てる - 2831063, # てく - 2029030, # ものの - 2568020, # せる - 900000, # たそう (custom) - 11553382, # となら (continuative of となる "to become") — always と+なら - 11566952, # となら (continuative of となる "to neighbor") — always と+なら - 2862334, # 振り出しに戻る - prefer component words (振り出し + に + 戻る) - 1881690, # 写真を撮る - prefer component words (写真 + を + 撮る) - 2860921, # 家を出る - prefer component words (家 + を + 出る) - 2862924, # 悪口(あっこう) - prefer わるぐち reading (seq 1575730) - 10551045, # 為し(なし) - prefer 無し adjective (seq 1529560) - 10456682, # 成し(なし) - prefer 無し adjective (seq 1529560) - 1465580, # 入る(いる) - prefer はいる reading (seq 1465590) - 1751450, # 精霊(しょうりょう) - prefer せいれい reading (seq 1380230) - 2657130, # 止まる(とどまる) - prefer とまる reading (seq 1310620) - 2838238, # こうぜ(校是) - obscure word, clashes with こう+ぜ (volitional+particle) - 2260200, # にい(新) - uncommon kana reading, clashes with に+い verb patterns - 2254970, # にい(兄) - uncommon kana reading, clashes with に+い verb patterns - 1774770, # から傘(からかさ) - uncommon, clashes with から(because)+傘(umbrella) - 10253059, # うける(浮く potential) - always prefer 受ける (seq 1329590) reading -} - -# Final particles - words that only have meaning when they're final -# From ichiran's *final-prt* -FINAL_PRT: Set[int] = { - 2017770, # かい - 2425930, # なの - 2130430, # け っけ - 2029130, # ぞ - 2834812, # ぜ - 2718360, # がな - 2201380, # わい - 2722170, # のう - 2751630, # かいな -} - -# Semi-final particles - final, but also have other uses -# From ichiran's *semi-final-prt* (which includes *final-prt*) -SEMI_FINAL_PRT: Set[int] = FINAL_PRT | { - 2029120, # さ - 2086640, # し - 2029110, # な - 2029080, # ね - 2029100, # わ -} - -# Non-final particles - don't get final bonus -# From ichiran's *non-final-prt* -NON_FINAL_PRT: Set[int] = { - 2139720, # ん -} - -# Words that get no kanji break penalty -# From ichiran's *no-kanji-break-penalty* -NO_KANJI_BREAK_PENALTY: Set[int] = { - 1169870, # 飲む - 1198360, # 会議 - 1277450, # 好き - 2028980, # で - 1423000, # 着る - 1164690, # 一段 - 1587040, # 言う - 2827864, # なので -} - -# Expression entries that are valid dictionary items but too coarse for our -# segmentation goals. We prefer compositional analysis for these. -SUPPRESS_SINGLE_TOKEN_SEQS: Set[int] = { - 2825978, # でしょうか -> prefer でしょう + か -} - -# Per-entry score bonuses to prefer linguistically compositional analyses. -SEQ_SCORE_BONUS: Dict[int, int] = { - 10044695: 10, # でしょう (conjugated copula) over spurious 〜で + しょうか paths -} - -# Import conjugation constants from central location -from himotoki.constants import ( - CONJ_ADVERBIAL, CONJ_ADJECTIVE_STEM, CONJ_NEGATIVE_STEM, - CONJ_CAUSATIVE_SU, CONJ_ADJECTIVE_LITERARY, - CONJ_TYPE_NAMES, WEAK_CONJ_FORMS, SKIP_CONJ_FORMS, - CONJ_VOLITIONAL, CONJ_TE, CONJ_POTENTIAL, CONJ_IMPERATIVE, -) - -# Archaic words cache - populated on first use -_ARCHAIC_CACHE: Optional[Set[int]] = None - -# ============================================================================ -# LRU Cache Implementation -# ============================================================================ -# A simple LRU cache using OrderedDict to prevent memory leaks in long-running -# processes. This replaces the previous unbounded Dict + "clear half" approach. - -class LRUCache: - """ - A simple LRU (Least Recently Used) cache with a maximum size. - - Uses OrderedDict to maintain insertion/access order. When the cache - reaches capacity, the least recently used item is evicted. - - Thread-safety: This implementation is NOT thread-safe. For multi-threaded - use, wrap access with a lock. - """ - - def __init__(self, maxsize: int): - self.maxsize = maxsize - self._cache: OrderedDict = OrderedDict() - - def get(self, key, default=None): - """Get item, moving it to the end (most recently used).""" - if key in self._cache: - self._cache.move_to_end(key) - return self._cache[key] - return default - - def __contains__(self, key): - return key in self._cache - - def __getitem__(self, key): - self._cache.move_to_end(key) - return self._cache[key] - - def __setitem__(self, key, value): - if key in self._cache: - self._cache.move_to_end(key) - else: - if len(self._cache) >= self.maxsize: - # Evict oldest (first) item - self._cache.popitem(last=False) - self._cache[key] = value - - def __len__(self): - return len(self._cache) - - def keys(self): - return self._cache.keys() - - def clear(self): - self._cache.clear() - - -# Conjugation data cache - (seq, from_seq, tuple(conj_ids), tuple(texts)) -> List[ConjData] -_CONJ_DATA_CACHE: LRUCache = LRUCache(maxsize=2048) - -# POS cache - seq -> frozenset(posi) for per-seq caching -_POS_SEQ_CACHE: LRUCache = LRUCache(maxsize=4096) - -# UK (prefer kana) cache - frozenset(seqs) -> bool -_UK_CACHE: LRUCache = LRUCache(maxsize=1024) - -# Word lookup cache - (word, is_kana) -> List[WordMatch readings data] -_WORD_CACHE: LRUCache = LRUCache(maxsize=4096) - -# Entry cache - seq -> Entry (reduces individual lookups) -_ENTRY_CACHE: LRUCache = LRUCache(maxsize=4096) - - -# ============================================================================ -# Data Structures -# ============================================================================ - -@dataclass(slots=True) -class ConjData: - """ - Conjugation data for a word match. - Tracks the conjugation chain from conjugated form to root. - """ - seq: int # Conjugated entry seq - from_seq: int # Root entry seq - via: Optional[int] # Intermediate seq for secondary conjugations - prop: Optional[ConjProp] # Conjugation property - src_map: List[Tuple[str, str]] = field(default_factory=list) # (conjugated_text, source_text) pairs - - -@dataclass -class WordMatch: - """ - Represents a word found in the database. - Wraps either a KanjiText/KanaText ORM object OR a lightweight RawKana/KanjiReading. - """ - reading: Union[KanjiText, KanaText, RawKanaReading, RawKanjiReading] - conjugations: Optional[List[int]] = None # List of conjugation IDs, or :root marker - hinted: bool = False - # Cached properties for performance (avoid repeated isinstance checks) - _word_type: Optional[str] = field(default=None, repr=False) - _seq: Optional[int] = field(default=None, repr=False) - _text: Optional[str] = field(default=None, repr=False) - - def __post_init__(self): - # Pre-compute cached properties on creation - # Detect type: ORM objects have __table__, raw namedtuples don't - # For raw types: RawKanaReading has 'best_kanji', RawKanjiReading has 'best_kana' - if isinstance(self.reading, (KanjiText, KanaText)): - # ORM object - self._word_type = 'kanji' if isinstance(self.reading, KanjiText) else 'kana' - elif isinstance(self.reading, RawKanaReading): - self._word_type = 'kana' - elif isinstance(self.reading, RawKanjiReading): - self._word_type = 'kanji' - else: - # Fallback: check for best_kanji attribute (kana has it, kanji doesn't) - self._word_type = 'kana' if hasattr(self.reading, 'best_kanji') else 'kanji' - self._seq = self.reading.seq - self._text = self.reading.text - - @property - def seq(self) -> int: - return self._seq - - @property - def text(self) -> str: - return self._text - - @property - def common(self) -> Optional[int]: - return self.reading.common - - @property - def ord(self) -> int: - return self.reading.ord - - @property - def word_type(self) -> str: - """Returns 'kanji' or 'kana' based on reading type.""" - return self._word_type - - @property - def is_root(self) -> bool: - """True if this is marked as a root form (not conjugated).""" - return self.conjugations == 'root' - - @property - def is_compound(self) -> bool: - """Always False for simple WordMatch.""" - return False - - @property - def components(self) -> List[str]: - """Return empty list for simple words (no components).""" - return [] - - def __repr__(self): - return f"" - - -@dataclass -class CompoundWord: - """ - Compound word made of 2 or more words joined together. - - Ports ichiran's compound-text class from dict.lisp lines 608-670. - Compound words are created when a primary word is joined with a suffix - (e.g., 食べている = 食べ + て + いる). - - Abbreviation compounds (is_abbrev=True) are scored differently - they use - the original word's mora length instead of the abbreviated compound length. - This matches Ichiran's proxy-text behavior for abbreviation suffixes. - """ - text: str # Full compound text - kana: str # Full kana reading - primary: WordMatch # Primary word (the main content word) - words: List[WordMatch] # All words in the compound - score_mod: Union[float, List[float]] = 0.0 # Score modifier(s) - score_base: Optional[WordMatch] = None # Base for scoring (usually primary) - is_abbrev: bool = False # True for abbreviation compounds (e.g., nai-x: ず/ざる/ぬ) - - @property - def seq(self) -> int: - """Returns primary seq for compound (not a list).""" - return self.primary.seq - - @property - def is_compound(self) -> bool: - """Always True for CompoundWord.""" - return True - - @property - def components(self) -> List[str]: - """Return component texts.""" - return [w.text for w in self.words] - - @property - def reading(self): - """Reading from primary word (for compatibility with calc_score).""" - return self.primary.reading - - @property - def is_root(self) -> bool: - """Compound words are never roots.""" - return False - - @property - def common(self) -> Optional[int]: - """Common rating from primary word.""" - return self.primary.common - - @property - def ord(self) -> int: - """Ord from primary word.""" - return self.primary.ord - - @property - def word_type(self) -> str: - """Word type from primary word.""" - return self.primary.word_type - - @property - def conjugations(self) -> Optional[List[int]]: - """Conjugations from last word in compound.""" - if self.words: - return self.words[-1].conjugations - return None - - @conjugations.setter - def conjugations(self, value): - """Set conjugations on last word.""" - if self.words: - self.words[-1].conjugations = value - - def get_score_base(self) -> WordMatch: - """Get the base word for scoring.""" - return self.score_base or self.primary - - def get_conjugation_info(self, session: 'Session') -> Dict[str, Any]: - """ - Get conjugation info from final word. - - Extracts conj_type, neg, fml, and source_text from the final word - in the compound. - - Args: - session: Database session for looking up conjugation data - - Returns: - Dict with keys: conj_type, neg, fml, source_text - """ - if not self.words: - return { - 'conj_type': None, - 'neg': False, - 'fml': False, - 'source_text': None, - } - - last_word = self.words[-1] - conj_data = get_word_conj_data(session, last_word) - - if not conj_data: - return { - 'conj_type': None, - 'neg': False, - 'fml': False, - 'source_text': None, - } - - # Get the first conjugation data entry - cd = conj_data[0] - prop = cd.prop - - # Get conj_type name - conj_type = None - neg = False - fml = False - if prop: - conj_type = prop.conj_type - neg = prop.neg if prop.neg else False - fml = prop.fml if prop.fml else False - - # Get source_text from src_map - source_text = None - if cd.src_map: - for text, src_text in cd.src_map: - if text == last_word.text: - source_text = src_text - break - - return { - 'conj_type': conj_type, - 'neg': neg, - 'fml': fml, - 'source_text': source_text, - } - - def __repr__(self): - return f"" - - -def adjoin_word( - word1: Union[WordMatch, 'CompoundWord'], - word2: WordMatch, - text: Optional[str] = None, - kana: Optional[str] = None, - score_mod: float = 0.0, - score_base: Optional[WordMatch] = None, - is_abbrev: bool = False, -) -> CompoundWord: - """ - Create compound word from 2 words. - - Ports ichiran's adjoin-word from dict.lisp lines 632-654. - - Args: - word1: Primary word or existing compound - word2: Word to append - text: Override text (default: concatenate both texts) - kana: Override kana (default: concatenate both kanas) - score_mod: Score modifier for this join - score_base: Base for scoring - is_abbrev: True if this is an abbreviation suffix (affects scoring) - - Returns: - CompoundWord combining both words - """ - # Default text and kana by concatenation - if text is None: - text = word1.text + word2.text - if kana is None: - # Derive kana from each word's kana/reading - def get_word_kana(w): - if isinstance(w, CompoundWord): - return w.kana - # For WordMatch, get kana from reading - if hasattr(w, 'reading'): - # KanjiText/RawKanjiReading has text=kanji, best_kana=kana reading - # KanaText/RawKanaReading has text=kana directly - if isinstance(w.reading, (KanjiText, RawKanjiReading)): - return w.reading.best_kana or w.reading.text - elif hasattr(w.reading, 'text'): - return w.reading.text - return w.text - kana = get_word_kana(word1) + get_word_kana(word2) - - if isinstance(word1, CompoundWord): - # Append to existing compound - word1.text = text - word1.kana = kana - word1.words = word1.words + [word2] - # Accumulate score_mod - if isinstance(word1.score_mod, list): - word1.score_mod = [score_mod] + word1.score_mod - else: - word1.score_mod = [score_mod, word1.score_mod] - # Mark as abbreviation if this join is an abbreviation - if is_abbrev: - word1.is_abbrev = True - return word1 - else: - # Create new compound from two simple words - return CompoundWord( - text=text, - kana=kana, - primary=word1, - words=[word1, word2], - score_mod=score_mod, - score_base=score_base, - is_abbrev=is_abbrev, - ) - - -@dataclass(slots=True) -class Segment: - """ - A segment representing a word match within a string. - Contains position information and scoring. - """ - start: int # Start position in source string - end: int # End position in source string - word: WordMatch - score: float = 0.0 - info: Dict[str, Any] = field(default_factory=dict) - text: Optional[str] = None # Cached text - top: bool = False # Whether this is the top segment in its list - # Filter result cache: filter_id -> bool result - # This avoids re-running expensive filters on the same segment - _filter_cache: Optional[Dict[int, bool]] = None - - def get_text(self) -> str: - if self.text is None: - self.text = self.word.text - return self.text - - def get_filter_result(self, filter_id: int) -> Optional[bool]: - """Get cached filter result if available.""" - if self._filter_cache is None: - return None - return self._filter_cache.get(filter_id) - - def set_filter_result(self, filter_id: int, result: bool) -> None: - """Cache a filter result.""" - if self._filter_cache is None: - self._filter_cache = {} - self._filter_cache[filter_id] = result - - def __repr__(self): - return f"" - - -@dataclass(slots=True) -class SegmentList: - """ - A list of segments at the same position. - Contains multiple possible interpretations for a substring. - """ - segments: List[Segment] - start: int - end: int - top: Any = None # TopArray for path finding - matches: int = 0 # Total number of matches found - - def __repr__(self): - return f"" - - -# ============================================================================ -# Length Coefficient Functions -# ============================================================================ - -def length_multiplier(length: int, power: float, len_lim: int) -> float: - """ - Calculate length multiplier: len^power until len_lim, linear after. - - Args: - length: Word length in mora - power: Exponent to use - len_lim: Limit after which growth becomes linear - - Returns: - The multiplier value - """ - if length <= len_lim: - return length ** power - return length * (len_lim ** (power - 1)) - - -def length_multiplier_coeff(length: int, coeff_class: str) -> int: - """ - Get length multiplier from coefficient sequence. - - Args: - length: Word length in mora (1-based) - coeff_class: One of 'strong', 'weak', 'tail', 'ltail' - - Returns: - The coefficient value for this length - """ - coeffs = LENGTH_COEFF_SEQUENCES.get(coeff_class) - if not coeffs: - return length - - if 0 < length < len(coeffs): - return coeffs[length] - - # Linear extrapolation for lengths beyond the table - last_coeff = coeffs[-1] - last_idx = len(coeffs) - 1 - return length * (last_coeff // last_idx) if last_idx > 0 else length - - -# ============================================================================ -# Database Lookup Functions -# ============================================================================ - -def find_word( - session: Session, - word: str, - root_only: bool = False, -) -> List[WordMatch]: - """ - Find words matching the given text in the database. - - Args: - session: Database session - word: Text to search for - root_only: If True, only return root entries (not conjugations) - - Returns: - List of WordMatch objects - """ - if len(word) > MAX_WORD_LENGTH: - return [] - - # Determine which table to search based on word content - if is_kana(word): - table = KanaText - else: - table = KanjiText - - if root_only: - # Join with entry to filter by root_p - query = ( - select(table) - .join(Entry, table.seq == Entry.seq) - .where(and_(table.text == word, Entry.root_p == True)) - ) - else: - query = select(table).where(table.text == word) - - results = session.execute(query).scalars().all() - results = [r for r in results if r.seq not in SUPPRESS_SINGLE_TOKEN_SEQS] - return [WordMatch(reading=r) for r in results] - - -def find_word_as_hiragana( - session: Session, - word: str, - exclude_seqs: Optional[Set[int]] = None, -) -> List[WordMatch]: - """ - Find words by converting katakana to hiragana and searching. - - Args: - session: Database session - word: Text to search for (may contain katakana) - exclude_seqs: Seqs to exclude from results - - Returns: - List of WordMatch objects with proxy text - """ - hiragana = as_hiragana(word) - if hiragana == word: - return [] - - words = find_word(session, hiragana, root_only=True) - - if exclude_seqs: - words = [w for w in words if w.seq not in exclude_seqs] - - # TODO: Create proxy text objects that preserve original form - return words - - -def find_word_full( - session: Session, - word: str, - as_hiragana_lookup: bool = False, - counter: Union[bool, int] = False, -) -> List[WordMatch]: - """ - Full word lookup with multiple strategies. - - Args: - session: Database session - word: Text to search for - as_hiragana_lookup: If True, also try converting katakana to hiragana - counter: If True, look for counter patterns - - Returns: - List of WordMatch objects - """ - simple_words = find_word(session, word) - results = list(simple_words) - - # Add suffix lookup (find_word_suffix) - from himotoki.suffixes import find_word_suffix - suffix_words = find_word_suffix(session, word, matches=simple_words) - results.extend(suffix_words) - - if as_hiragana_lookup: - exclude = {w.seq for w in simple_words} - results.extend(find_word_as_hiragana(session, word, exclude)) - - # TODO: Add counter lookup - - return results - - -# ============================================================================ -# Conjugation-based Lookup Functions -# ============================================================================ - -def find_word_with_conj_prop( - session: Session, - word: str, - filter_fn: callable, - allow_root: bool = False, -) -> List[WordMatch]: - """ - Find words matching text with conjugation property filter. - - Ports ichiran's find-word-with-conj-prop from dict-grammar.lisp. - - Args: - session: Database session - word: Text to search for - filter_fn: Function to filter ConjData objects - allow_root: If True, also return root forms - - Returns: - List of WordMatch objects with conjugation IDs set - """ - results = [] - for match in find_word_full(session, word): - conj_data = get_word_conj_data(session, match) - conj_data_filtered = [cd for cd in conj_data if filter_fn(cd)] - # Use conj_id (foreign key to Conjugation.id), not prop.id - # This matches ichiran's (conj-id (conj-data-prop cdata)) - conj_ids = [cd.prop.conj_id if cd.prop else None for cd in conj_data_filtered] - conj_ids = [cid for cid in conj_ids if cid is not None] - - if conj_data_filtered or (not conj_data and allow_root): - match.conjugations = conj_ids if conj_ids else None - results.append(match) - - return results - - -def find_word_with_conj_type( - session: Session, - word: str, - *conj_types: int, -) -> List[WordMatch]: - """ - Find words matching text with specific conjugation types. - - Ports ichiran's find-word-with-conj-type from dict-grammar.lisp. - - Args: - session: Database session - word: Text to search for - *conj_types: Conjugation type IDs to match - - Returns: - List of WordMatch objects - """ - def filter_fn(cdata: ConjData) -> bool: - if cdata.prop and hasattr(cdata.prop, 'conj_type'): - return cdata.prop.conj_type in conj_types - return False - - return find_word_with_conj_prop(session, word, filter_fn) - - -# Blocklist for spurious conjugation relationships in the database -# These are (seq, from_seq) pairs that should NOT be treated as conjugations -# because the seq entry is primarily a different word (e.g., particle) that -# happens to have the same reading as a conjugated form -# Example: Particle よ (seq 2029090) looks like adjective stem of いい (seq 2820690) -# but standalone よ should be parsed as particle, not adjective stem -BLOCKED_CONJUGATIONS = { - (2029090, 2820690), # よ (particle) ← いい (adj-ix adjective stem) -} - - -def get_conj_data( - session: Session, - seq: int, - from_seq: Optional[int] = None, - conj_ids: Optional[List[int]] = None, - texts: Optional[List[str]] = None, -) -> List[ConjData]: - """ - Get conjugation data for an entry. - - Args: - session: Database session - seq: Entry sequence number - from_seq: If provided, only get conjugations from this source - conj_ids: If provided, only get these specific conjugation IDs - texts: If provided, only get conjugations for these texts - - Returns: - List of ConjData objects - """ - global _CONJ_DATA_CACHE - - # Create cache key from immutable inputs - cache_key = (seq, from_seq, tuple(sorted(conj_ids)) if conj_ids else None, - tuple(sorted(texts)) if texts else None) - - # Check cache first - if cache_key in _CONJ_DATA_CACHE: - return _CONJ_DATA_CACHE[cache_key] - - # Build query for conjugations - query = select(Conjugation).where(Conjugation.seq == seq) - - if from_seq is not None: - query = query.where(Conjugation.from_seq == from_seq) - if conj_ids: - query = query.where(Conjugation.id.in_(conj_ids)) - - conjugations = session.execute(query).scalars().all() - - # Filter out blocked conjugation relationships - conjugations = [c for c in conjugations if (c.seq, c.from_seq) not in BLOCKED_CONJUGATIONS] - - result = [] - for conj in conjugations: - # Get source readings - src_query = select(ConjSourceReading).where( - ConjSourceReading.conj_id == conj.id - ) - if texts: - src_query = src_query.where(ConjSourceReading.text.in_(texts)) - - src_readings = session.execute(src_query).scalars().all() - src_map = [(sr.text, sr.source_text) for sr in src_readings] - - if texts and not src_map: - continue - - # Get conjugation properties - props = session.execute( - select(ConjProp).where(ConjProp.conj_id == conj.id) - ).scalars().all() - - for prop in props: - result.append(ConjData( - seq=conj.seq, - from_seq=conj.from_seq, - via=conj.via, - prop=prop, - src_map=src_map, - )) - - # Cache result (LRUCache handles eviction automatically) - _CONJ_DATA_CACHE[cache_key] = result - - return result - - -def get_word_conj_data( - session: Session, - word: Union[WordMatch, CompoundWord], -) -> List[ConjData]: - """ - Get conjugation data for a word match. - - Ports ichiran's word-conj-data method from dict.lisp. - - For simple words, gets conjugation data from the word's seq. - For compound words, gets conjugation data from the last word. - - Args: - session: Database session - word: WordMatch or CompoundWord - - Returns: - List of ConjData objects - """ - if isinstance(word, CompoundWord): - # For compound words, get conj data from last word - if word.words: - return get_word_conj_data(session, word.words[-1]) - return [] - - # For simple WordMatch - seq = word.seq - conj_ids = word.conjugations if word.conjugations and word.conjugations != 'root' else None - - if isinstance(conj_ids, list): - return get_conj_data(session, seq, conj_ids=conj_ids, texts=[word.text]) - elif not word.is_root: - return get_conj_data(session, seq, texts=[word.text]) - else: - return [] - - -def get_conj_type_name( - session: Session, - word: Union[WordMatch, CompoundWord], -) -> Optional[str]: - """ - Get human-readable conjugation type name for a word. - - Looks up the conjugation data for the word and returns the - human-readable name from CONJ_TYPE_NAMES mapping. - - Args: - session: Database session - word: WordMatch or CompoundWord - - Returns: - Human-readable conjugation type name (e.g., "Past (~ta)", "Conjunctive (~te)") - or None if no conjugation data found - """ - conj_data = get_word_conj_data(session, word) - if not conj_data: - return None - - # Get the first conjugation data entry - cd = conj_data[0] - if cd.prop and cd.prop.conj_type: - return CONJ_TYPE_NAMES.get(cd.prop.conj_type) - - return None - - -def get_conj_neg( - session: Session, - word: Union[WordMatch, CompoundWord], -) -> bool: - """ - Get whether a word is in negative form. - - Args: - session: Database session - word: WordMatch or CompoundWord - - Returns: - True if the word is in negative form, False otherwise - """ - conj_data = get_word_conj_data(session, word) - if not conj_data: - return False - - cd = conj_data[0] - if cd.prop: - return bool(cd.prop.neg) - - return False - - -def get_conj_fml( - session: Session, - word: Union[WordMatch, CompoundWord], -) -> bool: - """ - Get whether a word is in formal/polite form. - - Args: - session: Database session - word: WordMatch or CompoundWord - - Returns: - True if the word is in formal form, False otherwise - """ - conj_data = get_word_conj_data(session, word) - if not conj_data: - return False - - cd = conj_data[0] - if cd.prop: - return bool(cd.prop.fml) - - return False - - -def get_source_text( - session: Session, - word: Union[WordMatch, CompoundWord], -) -> Optional[str]: - """ - Get source text (dictionary form) for a conjugated word. - - Looks up the conjugation data and finds the matching src_map entry - to return the source text (e.g., "だ" for "で", "食べる" for "食べた"). - - Args: - session: Database session - word: WordMatch or CompoundWord - - Returns: - Source text (dictionary form) or None if not found - """ - conj_data = get_word_conj_data(session, word) - if not conj_data: - return None - - # Get the word's text to match against src_map - word_text = word.text - - # Search through all conjugation data entries - for cd in conj_data: - if cd.src_map: - for text, src_text in cd.src_map: - if text == word_text: - return src_text - - return None - - -# ============================================================================ -# Batch Preloading for Performance -# ============================================================================ - -def preload_scoring_caches(session: Session, seqs: Set[int]) -> None: - """ - Batch preload all caches needed for calc_score. - - This dramatically reduces cold-start latency by: - 1. Batch loading Entry objects - 2. Batch preloading UK (prefer-kana) status - 3. Batch preloading POS tags - - Call this before scoring a batch of segments. - - Args: - session: Database session - seqs: Set of seq numbers to preload - """ - global _ENTRY_CACHE, _UK_CACHE, _POS_SEQ_CACHE - - if not seqs: - return - - # Filter to seqs not already in entry cache - missing_seqs = seqs - set(_ENTRY_CACHE.keys()) - - if missing_seqs: - # Batch load entries - entries = session.execute( - select(Entry).where(Entry.seq.in_(missing_seqs)) - ).scalars().all() - - # LRUCache handles eviction automatically - for entry in entries: - _ENTRY_CACHE[entry.seq] = entry - - # Find seqs not yet in UK cache - uk_missing = {seq for seq in seqs if frozenset([seq]) not in _UK_CACHE} - - if uk_missing: - # Single batch query for all UK statuses - uk_seqs = set(session.execute( - select(SenseProp.seq) - .where(and_( - SenseProp.seq.in_(uk_missing), - SenseProp.tag == 'misc', - SenseProp.text == 'uk' - )) - .distinct() - ).scalars().all()) - - # Cache results for all checked seqs - for seq in uk_missing: - cache_key = frozenset([seq]) - _UK_CACHE[cache_key] = seq in uk_seqs - - # Find seqs not yet in POS cache (now using _POS_SEQ_CACHE) - pos_missing = {seq for seq in seqs if seq not in _POS_SEQ_CACHE} - - if pos_missing: - # Build archaic senses subquery once - arch_misc = {'arch', 'obsc', 'rare'} - arch_senses = ( - select(SenseProp.sense_id) - .where(and_( - SenseProp.tag == 'misc', - SenseProp.text.in_(arch_misc) - )) - ) - - # Batch query for all POS tags (excluding archaic senses) - pos_results = session.execute( - select(SenseProp.seq, SenseProp.text) - .where(and_( - SenseProp.seq.in_(pos_missing), - SenseProp.tag == 'pos', - ~SenseProp.sense_id.in_(arch_senses) - )) - ).all() - - # Group results by seq - seq_to_posi: Dict[int, Set[str]] = {seq: set() for seq in pos_missing} - for seq, pos_text in pos_results: - seq_to_posi[seq].add(pos_text) - - # LRUCache handles eviction automatically - for seq, posi in seq_to_posi.items(): - _POS_SEQ_CACHE[seq] = frozenset(posi) - - -def get_cached_entry(session: Session, seq: int) -> Optional[Entry]: - """ - Get Entry from cache or database. - Uses _ENTRY_CACHE for faster repeated lookups. - """ - global _ENTRY_CACHE - - if seq in _ENTRY_CACHE: - return _ENTRY_CACHE[seq] - - entry = session.get(Entry, seq) - - # LRUCache handles eviction automatically - if entry: - _ENTRY_CACHE[seq] = entry - - return entry - - -# ============================================================================ -# Archaic Word Detection -# ============================================================================ - -def build_archaic_cache(session: Session) -> Set[int]: - """ - Build cache of archaic/obsolete/rare word seqs. - From ichiran's *is-arch-cache*. - - Words where ALL senses are marked arch/obsc/rare are considered archaic. - A word with even one non-archaic sense is NOT considered archaic. - """ - arch_misc = {'arch', 'obsc', 'rare'} - - # Find seqs where EVERY sense has an arch/obsc/rare tag - # This is the ichiran logic: - # SELECT sense.seq FROM sense - # LEFT JOIN sense_prop sp ON (... AND sp.text IN ('arch', 'obsc', 'rare')) - # GROUP BY sense.seq HAVING EVERY(sp.id IS NOT NULL) - # - # In SQLAlchemy, we do this by: - # 1. Get all (seq, sense_id) pairs with their arch tag status - # 2. Group by seq and check that all senses have the arch tag - - from sqlalchemy import func, case, literal_column - from himotoki.db.models import Sense - - # Subquery: for each sense, is it archaic? - arch_tag_subq = ( - select(SenseProp.sense_id) - .where(and_( - SenseProp.tag == 'misc', - SenseProp.text.in_(arch_misc) - )) - ) - - # Main query: find seqs where ALL senses are in arch_tag_subq - # We count total senses and archaic senses per seq, keep only where they match - query = ( - select(Sense.seq) - .group_by(Sense.seq) - .having( - func.count(Sense.id) == func.sum( - case((Sense.id.in_(arch_tag_subq), 1), else_=0) - ) - ) - ) - arch_seqs = set(session.execute(query).scalars().all()) - - # Also add conjugations derived from archaic words - if arch_seqs: - conj_query = ( - select(Conjugation.seq) - .where(Conjugation.from_seq.in_(arch_seqs)) - .distinct() - ) - conj_seqs = set(session.execute(conj_query).scalars().all()) - arch_seqs |= conj_seqs - - return arch_seqs - - -def is_arch(session: Session, seq_set: Set[int]) -> bool: - """ - Check if all seqs in seq_set are archaic/obsolete/rare. - Uses a cached set of archaic word seqs. - """ - global _ARCHAIC_CACHE - if _ARCHAIC_CACHE is None: - _ARCHAIC_CACHE = build_archaic_cache(session) - - return all(seq in _ARCHAIC_CACHE for seq in seq_set) - - -def is_prefer_kana(session: Session, seq_set: List[int]) -> bool: - """ - Check if entries have 'uk' (usually written in kana) misc tag. - Cached for performance. - """ - global _UK_CACHE - - cache_key = frozenset(seq_set) - if cache_key in _UK_CACHE: - return _UK_CACHE[cache_key] - - result = session.execute( - select(SenseProp) - .where(and_( - SenseProp.seq.in_(seq_set), - SenseProp.tag == 'misc', - SenseProp.text == 'uk' - )) - ).scalars().first() is not None - - # LRUCache handles eviction automatically - _UK_CACHE[cache_key] = result - - return result - - -def get_non_arch_posi(session: Session, seq_set: Set[int]) -> Set[str]: - """ - Get part-of-speech tags for entries, excluding archaic senses. - From ichiran's get-non-arch-posi. - - Uses per-seq caching for better hit rate when seq_sets overlap. - """ - global _POS_SEQ_CACHE - - # Check if we have all seqs cached - all_cached = all(seq in _POS_SEQ_CACHE for seq in seq_set) - - if all_cached: - # Combine cached results - result = set() - for seq in seq_set: - result |= _POS_SEQ_CACHE[seq] - return result - - # Find which seqs need to be fetched - missing_seqs = {seq for seq in seq_set if seq not in _POS_SEQ_CACHE} - - if missing_seqs: - arch_misc = {'arch', 'obsc', 'rare'} - - # Subquery to find sense_ids with archaic props - arch_senses = ( - select(SenseProp.sense_id) - .where(and_( - SenseProp.tag == 'misc', - SenseProp.text.in_(arch_misc) - )) - ) - - # Batch query for all missing seqs - results = session.execute( - select(SenseProp.seq, SenseProp.text) - .where(and_( - SenseProp.seq.in_(missing_seqs), - SenseProp.tag == 'pos', - ~SenseProp.sense_id.in_(arch_senses) - )) - ).all() - - # Group by seq - seq_posi: Dict[int, Set[str]] = {seq: set() for seq in missing_seqs} - for seq, pos_text in results: - seq_posi[seq].add(pos_text) - - # LRUCache handles eviction automatically - for seq, posi in seq_posi.items(): - _POS_SEQ_CACHE[seq] = frozenset(posi) - - # Combine results from cache - result = set() - for seq in seq_set: - if seq in _POS_SEQ_CACHE: - result |= _POS_SEQ_CACHE[seq] - - return result - - -# ============================================================================ -# Conjugation Form Testing -# ============================================================================ - -def matches_conj_form(prop: ConjProp, forms: List[Tuple]) -> bool: - """ - Check if a conjugation property matches any of the weak/skip forms. - From ichiran's test-conj-prop. - - Args: - prop: ConjProp object to test - forms: List of (conj_type, neg, fml) or (pos, conj_type, neg, fml) tuples - where None means "any value matches" - - Returns: - True if prop matches any form pattern - """ - for form in forms: - if len(form) == 3: - # (conj_type, neg, fml) format - pattern = [prop.conj_type, prop.neg, prop.fml] - if all( - r is None or l == r - for l, r in zip(pattern, form) - ): - return True - elif len(form) == 4: - # (pos, conj_type, neg, fml) format - if form[0] == prop.pos: - pattern = [prop.conj_type, prop.neg, prop.fml] - if all( - r is None or l == r - for l, r in zip(pattern, form[1:]) - ): - return True - return False - - -def skip_by_conj_data(conj_data: List[ConjData]) -> bool: - """ - Check if conjugation data should be skipped entirely. - From ichiran's skip-by-conj-data. - - Returns True if ALL conjugation data matches skip patterns. - """ - if not conj_data: - return False - - return all( - cd.prop is not None and matches_conj_form(cd.prop, SKIP_CONJ_FORMS) - for cd in conj_data - ) - - -def is_weak_conj_form(conj_data: List[ConjData]) -> bool: - """ - Check if all conjugations are weak forms (don't contribute as much). - """ - if not conj_data: - return False - - return all( - cd.prop is not None and matches_conj_form(cd.prop, WEAK_CONJ_FORMS) - for cd in conj_data - ) - - -# ============================================================================ -# Scoring Functions -# ============================================================================ - -def compare_common(c1: Optional[int], c2: Optional[int]) -> bool: - """ - Compare two commonness values. - Lower is better, 0 is special (very common), None is worst. - - Returns True if c1 should be sorted before c2. - """ - if c2 is None: - return c1 is not None - if c2 == 0: - return c1 is not None and c1 > 0 - if c1 is not None and c1 > 0: - return c1 < c2 - return False - - -def kanji_break_penalty( - kanji_break: List[int], - score: float, - info: Optional[Dict] = None, - text: str = "", - use_length: Optional[int] = None, - score_mod: float = 0, -) -> float: - """ - Apply penalty for breaks within kanji sequences. - - Args: - kanji_break: List of positions where kanji are broken - score: Current score - info: Score info dict - text: Word text - use_length: Context length if available - score_mod: Score modifier - - Returns: - Adjusted score - """ - if not kanji_break: - return score - - # Determine break position type - end = 'both' if len(kanji_break) > 1 else ( - 'beg' if kanji_break[0] == 0 else 'end' - ) - - bonus = 0 - ratio = 2 - posi = info.get('posi', []) if info else [] - - if info: - seq_set = info.get('seq_set', set()) - - # Check for no-penalty words - if seq_set & NO_KANJI_BREAK_PENALTY: - return score - - # Check for special す break - if end == 'beg' and text.startswith('す'): - return score - - # Adjust bonus based on POS - if end == 'beg' and 'num' in posi: - bonus += 5 - elif end == 'beg' and ('suf' in posi or 'n-suf' in posi): - bonus += 10 - elif end == 'end' and 'pref' in posi: - bonus += 12 - - if score >= SCORE_CUTOFF: - return max(SCORE_CUTOFF, (score // ratio) + bonus) - return score - - -def calc_score( - session: Session, - word: Union[WordMatch, CompoundWord], - final: bool = False, - use_length: Optional[int] = None, - score_mod: float = 0, - kanji_break: Optional[List[int]] = None, -) -> Tuple[float, Dict[str, Any]]: - """ - Calculate score for a word match. - - This is the core scoring algorithm ported from ichiran's calc-score (dict.lisp lines 777-983). - - The scoring system considers: - - Word type (kanji vs kana) - - Commonness ranking - - Part of speech (particles get special handling) - - Primary/secondary reading status - - Conjugation status and type - - Word length - - Context length (use_length) - - Kanji break penalties - - Archaic word detection - - Args: - session: Database session - word: WordMatch or CompoundWord object to score - final: True if this is at the end of the text - use_length: Context length for scoring - score_mod: Score modifier (for compound words) - kanji_break: List of kanji break positions - - Returns: - Tuple of (score, info_dict) - """ - # Handle compound words by scoring the base word with compound properties - # From ichiran dict.lisp lines 782-794 - if isinstance(word, CompoundWord): - base_word = word.get_score_base() - compound_score_mod = word.score_mod - - # Both regular compounds and abbreviation compounds use the compound - # text's mora length for use_length. This creates a penalty when the - # compound is shorter than the base word (negative difference in - # length_multiplier_coeff). - # - # For abbreviations like とまず (3 mora) from とまない (4 mora): - # use_length=3, base word len=4, difference=-1 → penalty applied - compound_use_length = mora_length(word.text) - - score, info = calc_score( - session, base_word, - use_length=compound_use_length, - score_mod=compound_score_mod, - ) - - # Add compound word conjugation data - info['conj'] = get_word_conj_data(session, word) - - # Apply kanji break penalty if needed - if kanji_break: - score = kanji_break_penalty( - kanji_break, score, - info=info, text=base_word.text, - use_length=compound_use_length, score_mod=compound_score_mod - ) - - return score, info - - # Check for counter mode (CounterText objects) - ctr_mode = _is_counter_text(word) - - reading = word.reading - text = word.text - seq = word.seq - ord_val = word.ord - common = word.common - - # Fast path: Early termination for skip words (before any DB lookups) - # This avoids expensive entry lookups and conjugation data retrieval - if seq in SKIP_WORDS: - return 0, {} - - # Suppress coarse expression tokens when compositional segmentation exists. - if seq in SUPPRESS_SINGLE_TOKEN_SEQS: - return 0, {} - - # Fast path: Final particles only score at end of text - if not final and seq in FINAL_PRT: - return 0, {} - - # Get entry info - counters don't have entries - # Use cached entry lookup for performance - entry = None if ctr_mode else get_cached_entry(session, seq) - if not entry and not ctr_mode: - return 0, {} - - # Basic properties - score = 1 - prop_score = 0 - - kanji_p = word.word_type == 'kanji' - katakana_p = not kanji_p and count_char_class(text, 'katakana') > 0 - - n_kanji = count_char_class(text, 'kanji') - word_len = max(1, mora_length(text)) - - # Conjugation info - counters don't have conjugations - conj_only = False if ctr_mode else (word.conjugations is not None and word.conjugations != 'root') - root_p = ctr_mode or (not conj_only and entry and entry.root_p) - - # Get conjugation data - counters don't have conjugations - if ctr_mode: - conj_data = [] - elif conj_only and isinstance(word.conjugations, list): - conj_data = get_conj_data(session, seq, conj_ids=word.conjugations, texts=[text]) - elif not word.is_root: - conj_data = get_conj_data(session, seq, texts=[text]) - else: - conj_data = [] - - # Handle secondary conjugations (via forms) - # If this is nil, delete all secondary conjugations from conj data - secondary_conj_p = False - if conj_data: - if all(cd.via for cd in conj_data): - secondary_conj_p = True - else: - # Remove secondary conjugations from data - conj_data = [cd for cd in conj_data if not cd.via] - - conj_of = [cd.from_seq for cd in conj_data] - conj_props = [cd.prop for cd in conj_data if cd.prop] - conj_types = [cp.conj_type for cp in conj_props] - - # conj_types_p: True if not all conjugations are weak forms - # From ichiran: "weak" forms don't contribute to scoring as much - conj_types_p = ( - root_p or - use_length is not None or - not all(matches_conj_form(prop, WEAK_CONJ_FORMS) for prop in conj_props if prop) - ) - - # Get part-of-speech info - seq_set = {seq} | set(conj_of) if seq else set() - sp_seq_set = [seq] if (seq and root_p and not use_length) else list(seq_set) - - # For counters, use 'ctr' as the part of speech - if ctr_mode: - prefer_kana = False - is_arch_p = False - posi = {'ctr'} - else: - # Check for prefer kana (uk - usually written in kana) - cached - prefer_kana = is_prefer_kana(session, sp_seq_set) - - # Check if all entries are archaic - is_arch_p = is_arch(session, set(sp_seq_set)) - - # Get part-of-speech (excluding archaic senses) - cached - posi = get_non_arch_posi(session, seq_set) - - # Common properties - common_p = common is not None - common_of = common - particle_p = 'prt' in posi - semi_final_particle_p = seq in SEMI_FINAL_PRT - non_final_particle_p = seq in NON_FINAL_PRT - pronoun_p = 'pn' in posi - cop_da_p = bool(seq_set & COPULAE) - - # Length classification (ichiran lines 836-844) - # More complex logic based on various conditions - if kanji_p and not prefer_kana: - if (root_p and not conj_data) or (use_length and 13 in conj_types): - len_threshold = 2 - elif common_p and common and 0 < common < 10: - len_threshold = 2 - elif {3, 9} & set(conj_types) and not use_length: - len_threshold = 4 - else: - len_threshold = 3 - else: - if common_p and common and 0 < common < 10: - len_threshold = 2 - elif {3, 9} & set(conj_types) and not use_length: - len_threshold = 4 - else: - len_threshold = 3 - - long_p = word_len > len_threshold - - # no_common_bonus conditions - no_common_bonus = ( - particle_p or - not conj_types_p or - (not long_p and posi == {'int'}) - ) - - use_length_bonus = 0 - - # Check for skip words in conjugation chain - # Only check the full seq_set (including conjugation sources) for DIRECT word matches, - # not when scoring a base word from a compound (use_length is set for compounds). - # This allows abbreviation compounds like とどまらず to score properly even though - # their source verb (とどまる) is in SKIP_WORDS. - if use_length is None and seq_set & SKIP_WORDS: - return 0, {} - elif seq in SKIP_WORDS: - # Always skip if the direct seq is in skip words - return 0, {} - if not root_p and skip_by_conj_data(conj_data): - return 0, {} - - # Handle inherited commonness and ord from conjugation source (ichiran lines 859-870) - # This MUST happen BEFORE primary_p determination so that ord_val is correct - if conj_data and not (ord_val == 0 and common_p): - orig_texts = get_original_text_data(session, word, conj_data) - if orig_texts: - if not common_p: - conj_of_common = [c for c, o in orig_texts if c is not None] - if conj_of_common: - common = 0 - common_p = True - # Get the "best" common value - common_of = sorted(conj_of_common, key=lambda c: (c or 1000, c == 0))[0] - - # Update ord if conjugated form has lower ord - conj_of_ord = min(o for c, o in orig_texts) - if conj_of_ord < ord_val: - ord_val = conj_of_ord - - # Primary reading check - now with archaic consideration - # Pass ord_val to use corrected ord from conjugation source - primary_p = False - if not is_arch_p: - primary_p = determine_primary_full( - session, entry, word, posi, common_p, kanji_p, - root_p, conj_data, prefer_kana, conj_types_p, cop_da_p, pronoun_p, - ord_override=ord_val - ) - - # Calculate base score (ichiran lines 890-925) - if primary_p: - if long_p: - score += 10 - elif secondary_conj_p and not kanji_p: - score += 2 - elif common_p and conj_types_p: - score += 5 - elif prefer_kana or not entry or entry.n_kanji == 0: - score += 3 - else: - score += 2 - - # Particle bonus (lines 896-902) - if particle_p and (final or not semi_final_particle_p): - score += 2 - if common_p: - score += 2 + word_len - if final and not non_final_particle_p: - if primary_p: - score += 5 - elif semi_final_particle_p: - score += 2 - - # Commonness bonus (lines 903-918) - if common_p and not no_common_bonus: - if secondary_conj_p and not use_length: - common_bonus = 4 if (kanji_p and primary_p) else 2 - elif long_p or cop_da_p or (root_p and (kanji_p or (primary_p and word_len > 2))): - if common == 0: - common_bonus = 10 - elif not primary_p: - common_bonus = max(15 - (common or 0), 10) - else: - common_bonus = max(20 - (common or 0), 10) - elif kanji_p: - common_bonus = 8 - elif primary_p: - common_bonus = 4 - elif word_len > 2 or (common and 0 < common < 10): - common_bonus = 3 - else: - common_bonus = 2 - - # Reduce bonus for continuative form (conj_type 10) - if common_bonus >= 10 and 10 in conj_types: - common_bonus -= 4 - - score += common_bonus - - # Length and kanji bonuses (lines 919-926) - if long_p: - score = max(word_len, score) - - if kanji_p: - score = max(3 if is_arch_p else 5, score) - if long_p and (n_kanji > 1 or word_len > 4): - score += 2 - - # Counter mode minimum score (ichiran line 926: (when ctr-mode (setf score (max 5 score)))) - if ctr_mode: - score = max(5, score) - - # Entry-specific score adjustments - score += SEQ_SCORE_BONUS.get(seq, 0) - - # Calculate prop_score and apply length multiplier (lines 927-937) - prop_score = score - length_class = 'strong' if (kanji_p or katakana_p) else 'weak' - score = prop_score * ( - length_multiplier_coeff(word_len, length_class) + - ((n_kanji - 1) * 5 if n_kanji > 1 else 0) - ) - - # Split scoring integration (ichiran lines 927-970) - # Check for split definition and apply split scoring - # Note: counters don't use split scoring (ichiran: (unless ctr-mode ...)) - split_info = None - if not ctr_mode: - from himotoki.splits import get_split - split_result = get_split(session, word, conj_of if conj_of else None) - - if split_result: - if ':score' in split_result.modifiers: - # Direct score addition mode - score += split_result.score_bonus - split_info = ('score', split_result.score_bonus) - elif ':pscore' in split_result.modifiers: - # Proportional score modification mode - import math - new_prop_score = max(1, prop_score + split_result.score_bonus) - score = math.ceil(score * new_prop_score / prop_score) if prop_score > 0 else score - prop_score = new_prop_score - split_info = ('pscore', split_result.score_bonus) - else: - # Standard split: sum of part scores + bonus - split_score = split_result.score_bonus - part_scores = [] - for i, part in enumerate(split_result.parts): - is_last = (i == len(split_result.parts) - 1) - # Calculate adjusted use_length for final part - part_use_length = None - if is_last and use_length: - # Subtract mora lengths of preceding parts - preceding_mora = sum( - mora_length(p.text) for p in split_result.parts[:-1] - ) - part_use_length = use_length - preceding_mora - - part_score, _ = calc_score( - session, part.reading, - final=final and is_last, - use_length=part_use_length, - score_mod=score_mod if is_last else 0, - ) - part_scores.append(part_score) - split_score += part_score - - score = split_score - split_info = ('split', split_result.score_bonus, part_scores) - - # Apply use_length bonus for context - if use_length: - tail_len = use_length - word_len - tail_class = 'ltail' if (word_len > 3 and (kanji_p or katakana_p)) else 'tail' - use_length_bonus = prop_score * length_multiplier_coeff(tail_len, tail_class) - - if score_mod: - use_length_bonus += apply_score_mod(score_mod, prop_score, tail_len) - - score += use_length_bonus - - # Build info dict - info = { - 'posi': list(posi), - 'seq_set': seq_set, - 'conj': conj_data, - 'common': common_of if common_p else None, - 'score_info': [prop_score, kanji_break, use_length_bonus, split_info], - 'kpcl': [kanji_p or katakana_p, primary_p, common_p, long_p], - } - - # Apply kanji break penalty - if kanji_break: - score = kanji_break_penalty( - kanji_break, score, - info=info, text=text, - use_length=use_length, score_mod=score_mod - ) - - return score, info - - -def determine_primary_full( - session: Session, - entry: Entry, - word: WordMatch, - posi: Set[str], - common_p: bool, - kanji_p: bool, - root_p: bool, - conj_data: List[ConjData], - prefer_kana: bool, - conj_types_p: bool, - cop_da_p: bool, - pronoun_p: bool, - ord_override: Optional[int] = None, -) -> bool: - """ - Full primary reading determination from ichiran lines 872-888. - More complete than the simple version. - - Args: - ord_override: If provided, use this ord value instead of word.ord. - This is used when the ord has been corrected based on - conjugation source data. - """ - if not entry: - return True - - # Use overridden ord if provided, otherwise use word.ord - ord_val = ord_override if ord_override is not None else word.ord - - # Prefer kana and this is kana reading - if prefer_kana and conj_types_p and not kanji_p: - if not entry.primary_nokanji: - return True - # Check nokanji flag on reading - if hasattr(word.reading, 'nokanji') and word.reading.nokanji: - return True - # Additional case: common hiragana word with ord=0 should be primary - # This handles words like きれい that are commonly written in kana - # but don't have the nokanji flag on the hiragana reading - if ord_val == 0 and common_p and (word.common == 0 or (word.common is not None and word.common < 10)): - return True - - # Primary if ord=0 or copula - if ord_val == 0 or cop_da_p: - if (kanji_p or conj_types_p) and ( - (kanji_p and not prefer_kana) or - (common_p and pronoun_p) or - entry.n_kanji == 0 - ): - return True - - # Special case: prefer_kana with kanji, ord=0, but uk is not for first sense - if prefer_kana and kanji_p and ord_val == 0: - # Check if uk prop is for ord=0 sense - first_sense_uk = session.execute( - select(SenseProp) - .join(Sense, SenseProp.sense_id == Sense.id) - .where(and_( - Sense.seq == entry.seq, - Sense.ord == 0, - SenseProp.tag == 'misc', - SenseProp.text == 'uk' - )) - ).scalars().first() - if not first_sense_uk: - return True - - return False - - -def get_original_text_data( - session: Session, - word: WordMatch, - conj_data: List[ConjData], -) -> List[Tuple[Optional[int], int]]: - """ - Get (common, ord) pairs from original (unconjugated) text. - - Ports ichiran's get-original-text* function from dict.lisp. - Extended from get_original_text_common to also return ord. - - For secondary conjugations (via forms), this function recursively - follows the conjugation chain to find the original source text. - - Args: - session: Database session - word: WordMatch object to get original text for - conj_data: List of ConjData objects for the word - - Returns: - List of (common, ord) tuples from the original source forms - """ - return _get_original_text_data_recursive(session, conj_data, [word.text]) - - -def _get_original_text_data_recursive( - session: Session, - conj_data: List[ConjData], - texts: List[str], -) -> List[Tuple[Optional[int], int]]: - """ - Recursive helper for get_original_text_data. - - Follows the conjugation chain through via forms to find the - ultimate source text and its properties. - - Args: - session: Database session - conj_data: List of ConjData objects - texts: List of text forms to look up - - Returns: - List of (common, ord) tuples from the original source forms - """ - result = [] - for cd in conj_data: - # Find matching source texts from src_map - src_texts = [] - for text, src_text in cd.src_map: - if text in texts: - src_texts.append(src_text) - - if not src_texts: - continue - - if cd.via is None: - # Direct conjugation - look up the source text in from_seq - for src_text in src_texts: - table = KanjiText if has_kanji(src_text) else KanaText - orig = session.execute( - select(table) - .where(and_(table.seq == cd.from_seq, table.text == src_text)) - ).scalars().first() - if orig: - result.append((orig.common, orig.ord)) - else: - # Secondary conjugation (via form) - recursively follow the chain - # Get conjugation data from via -> from_seq - via_conj_data = get_conj_data(session, cd.via, from_seq=cd.from_seq) - if via_conj_data: - result.extend(_get_original_text_data_recursive( - session, via_conj_data, src_texts - )) - - return result - - -def apply_score_mod( - score_mod: Union[int, float, callable, List], - score: float, - length: int, -) -> float: - """Apply score modifier.""" - if callable(score_mod): - return score_mod(score) - if isinstance(score_mod, list): - return sum(apply_score_mod(sm, score, length) for sm in score_mod) - return score * score_mod * length - - -def get_entry_posi(session: Session, seq_set: Set[int]) -> Set[str]: - """Get part-of-speech tags for entries.""" - query = ( - select(SenseProp.text) - .where(and_( - SenseProp.seq.in_(seq_set), - SenseProp.tag == 'pos' - )) - .distinct() - ) - results = session.execute(query).scalars().all() - return set(results) - - -def determine_primary( - session: Session, - entry: Entry, - word: WordMatch, - posi: Set[str], - common_p: bool, - kanji_p: bool, - root_p: bool, - conj_data: List[ConjData], -) -> bool: - """Determine if this is the primary reading for the entry.""" - if not entry: - return True - - # Check for uk (usually kana) preference - prefer_kana = session.execute( - select(SenseProp) - .where(and_( - SenseProp.seq == entry.seq, - SenseProp.tag == 'misc', - SenseProp.text == 'uk' - )) - ).scalars().first() is not None - - if prefer_kana and not kanji_p and word.ord == 0: - return True - - # Primary if ord=0 and kanji form or no kanji exists - if word.ord == 0: - if kanji_p or entry.n_kanji == 0: - return True - - # Check for pronoun with common reading - if common_p and 'pn' in posi and word.ord == 0: - return True - - return False - - -def get_original_text_common( - session: Session, - word: WordMatch, - conj_data: List[ConjData], -) -> Optional[int]: - """Get commonness from original (unconjugated) text.""" - for cd in conj_data: - for text, src_text in cd.src_map: - if text == word.text: - # Look up the source text - table = KanjiText if has_kanji(src_text) else KanaText - orig = session.execute( - select(table) - .where(and_(table.seq == cd.from_seq, table.text == src_text)) - ).scalars().first() - if orig and orig.common is not None: - return orig.common - return None - - -# ============================================================================ -# Segment Filtering Functions -# ============================================================================ - -def cull_segments(segments: List[Segment]) -> List[Segment]: - """ - Filter segments to remove low-scoring duplicates. - - Keeps segments scoring at least IDENTICAL_WORD_SCORE_CUTOFF of the max. - Sorts by score descending, then by commonness ascending. - """ - if not segments: - return segments - - # Sort by score descending, then by commonness ascending - # Note: common=0 is the best (most common), so we need to handle it specially - # since 0 is falsy in Python - def get_common_key(s): - common = s.info.get('common') if s.info else None - if common is None: - return float('inf') - return common - - segments = sorted( - segments, - key=lambda s: ( - -s.score, # Score descending (higher is better) - get_common_key(s), # Commonness ascending (lower is better) - ) - ) - - max_score = max(s.score for s in segments) - cutoff = max_score * IDENTICAL_WORD_SCORE_CUTOFF - - # Protect suffix compounds from culling — they carry grammar chain info - # that dict entries don't have, even when their score is lower - return [s for s in segments if s.score >= cutoff - or getattr(s.word, 'is_compound', False)] - - -def gen_score( - session: Session, - segment: Segment, - final: bool = False, - kanji_break: Optional[List[int]] = None, -) -> Segment: - """Generate score for a segment.""" - score, info = calc_score( - session, segment.word, - final=final, kanji_break=kanji_break - ) - segment.score = score - # Preserve counter flag if it was set - if segment.info and segment.info.get('counter'): - info['counter'] = True - segment.info = info - return segment - - -# ============================================================================ -# Path Finding Utilities -# ============================================================================ - -def gap_penalty(start: int, end: int) -> int: - """Calculate penalty for ungapped text.""" - return (end - start) * GAP_PENALTY diff --git a/himotoki/lookup/__init__.py b/himotoki/lookup/__init__.py new file mode 100644 index 0000000..c16af52 --- /dev/null +++ b/himotoki/lookup/__init__.py @@ -0,0 +1,136 @@ +""" +Word lookup and scoring module for himotoki. +Ports ichiran's dict.lisp word lookup and scoring functionality. + +This module provides: +- find_word: Database lookup for words +- calc_score: Complex scoring algorithm for word segments +- Segment/SegmentList: Data structures for word matches +- Length multipliers and scoring coefficients +""" + +# Types +from himotoki.types import ( + ConjData, + WordMatch, + CompoundWord, + adjoin_word, + Segment, + SegmentList, +) + +# Lookup constants +from himotoki.lookup.constants import ( + POS_WITH_CONJ_RULES, + SUPPRESS_SINGLE_TOKEN_SEQS, + CONJ_ADVERBIAL, + CONJ_ADJECTIVE_STEM, + CONJ_NEGATIVE_STEM, + CONJ_CAUSATIVE_SU, + CONJ_ADJECTIVE_LITERARY, + CONJ_TYPE_NAMES, + WEAK_CONJ_FORMS, + SKIP_CONJ_FORMS, + CONJ_VOLITIONAL, + CONJ_TE, + CONJ_POTENTIAL, + CONJ_IMPERATIVE, +) + +# Word lookup +from himotoki.lookup.find_word import ( + find_word, + find_word_as_hiragana, + find_word_full, + find_word_with_conj_prop, + find_word_with_conj_type, +) + +# Conjugation data +from himotoki.lookup.conj_data import ( + BLOCKED_CONJUGATIONS, + get_conj_data, + get_word_conj_data, + get_conj_type_name, + get_conj_neg, + get_conj_fml, + get_source_text, +) + +# Scoring (re-exported for backward compatibility) +from himotoki.scoring import ( + LRUCache, + _CONJ_DATA_CACHE, + _POS_SEQ_CACHE, + _UK_CACHE, + _WORD_CACHE, + _ENTRY_CACHE, + _ARCHAIC_CACHE, + clear_scoring_caches, + preload_scoring_caches, + get_cached_entry, + build_archaic_cache, + is_arch, + is_prefer_kana, + get_non_arch_posi, + MAX_WORD_LENGTH, + SCORE_CUTOFF, + GAP_PENALTY, + LENGTH_COEFF_SEQUENCES, + IDENTICAL_WORD_SCORE_CUTOFF, + SKIP_WORDS, + COPULAE, + NO_KANJI_BREAK_PENALTY, + FINAL_PRT, + SEMI_FINAL_PRT, + NON_FINAL_PRT, + SEQ_SCORE_BONUS, + length_multiplier, + length_multiplier_coeff, + matches_conj_form, + skip_by_conj_data, + is_weak_conj_form, + compare_common, + kanji_break_penalty, + calc_score, + determine_primary_full, + get_original_text_data, + apply_score_mod, + get_entry_posi, + determine_primary, + get_original_text_common, + cull_segments, + gen_score, + gap_penalty, +) + +__all__ = [ + # Types + 'ConjData', 'WordMatch', 'CompoundWord', 'adjoin_word', 'Segment', 'SegmentList', + # Constants + 'POS_WITH_CONJ_RULES', 'SUPPRESS_SINGLE_TOKEN_SEQS', + 'CONJ_ADVERBIAL', 'CONJ_ADJECTIVE_STEM', 'CONJ_NEGATIVE_STEM', + 'CONJ_CAUSATIVE_SU', 'CONJ_ADJECTIVE_LITERARY', + 'CONJ_TYPE_NAMES', 'WEAK_CONJ_FORMS', 'SKIP_CONJ_FORMS', + 'CONJ_VOLITIONAL', 'CONJ_TE', 'CONJ_POTENTIAL', 'CONJ_IMPERATIVE', + 'MAX_WORD_LENGTH', 'SCORE_CUTOFF', 'GAP_PENALTY', 'LENGTH_COEFF_SEQUENCES', + 'IDENTICAL_WORD_SCORE_CUTOFF', 'SKIP_WORDS', 'COPULAE', 'NO_KANJI_BREAK_PENALTY', + 'FINAL_PRT', 'SEMI_FINAL_PRT', 'NON_FINAL_PRT', 'SEQ_SCORE_BONUS', + # Lookup + 'find_word', 'find_word_as_hiragana', 'find_word_full', + 'find_word_with_conj_prop', 'find_word_with_conj_type', + 'BLOCKED_CONJUGATIONS', 'get_conj_data', 'get_word_conj_data', + 'get_conj_type_name', 'get_conj_neg', 'get_conj_fml', 'get_source_text', + # Scoring + 'LRUCache', '_CONJ_DATA_CACHE', '_POS_SEQ_CACHE', '_UK_CACHE', + '_WORD_CACHE', '_ENTRY_CACHE', '_ARCHAIC_CACHE', + 'clear_scoring_caches', + 'preload_scoring_caches', 'get_cached_entry', 'build_archaic_cache', + 'is_arch', 'is_prefer_kana', 'get_non_arch_posi', + 'length_multiplier', 'length_multiplier_coeff', + 'matches_conj_form', 'skip_by_conj_data', 'is_weak_conj_form', + 'compare_common', 'kanji_break_penalty', 'calc_score', + 'determine_primary_full', 'get_original_text_data', + 'apply_score_mod', 'get_entry_posi', 'determine_primary', + 'get_original_text_common', 'cull_segments', 'gen_score', 'gap_penalty', +] diff --git a/himotoki/lookup/conj_data.py b/himotoki/lookup/conj_data.py new file mode 100644 index 0000000..a2ec26e --- /dev/null +++ b/himotoki/lookup/conj_data.py @@ -0,0 +1,265 @@ +""" +Conjugation data lookup functions. +""" + +from typing import Optional, List, Union + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from himotoki.db.models import Conjugation, ConjProp, ConjSourceReading +from himotoki.types import ConjData, WordMatch, CompoundWord +from himotoki.scoring.caches import _CONJ_DATA_CACHE +from himotoki.lookup.constants import CONJ_TYPE_NAMES + +# Blocklist for spurious conjugation relationships in the database +# These are (seq, from_seq) pairs that should NOT be treated as conjugations +# because the seq entry is primarily a different word (e.g., particle) that +# happens to have the same reading as a conjugated form +# Example: Particle よ (seq 2029090) looks like adjective stem of いい (seq 2820690) +# but standalone よ should be parsed as particle, not adjective stem +BLOCKED_CONJUGATIONS = { + (2029090, 2820690), # よ (particle) ← いい (adj-ix adjective stem) +} + + +def get_conj_data( + session: Session, + seq: int, + from_seq: Optional[int] = None, + conj_ids: Optional[List[int]] = None, + texts: Optional[List[str]] = None, +) -> List[ConjData]: + """ + Get conjugation data for an entry. + + Args: + session: Database session + seq: Entry sequence number + from_seq: If provided, only get conjugations from this source + conj_ids: If provided, only get these specific conjugation IDs + texts: If provided, only get conjugations for these texts + + Returns: + List of ConjData objects + """ + global _CONJ_DATA_CACHE + + # Create cache key from immutable inputs + cache_key = (seq, from_seq, tuple(sorted(conj_ids)) if conj_ids else None, + tuple(sorted(texts)) if texts else None) + + # Check cache first + if cache_key in _CONJ_DATA_CACHE: + return _CONJ_DATA_CACHE[cache_key] + + # Build query for conjugations + query = select(Conjugation).where(Conjugation.seq == seq) + + if from_seq is not None: + query = query.where(Conjugation.from_seq == from_seq) + if conj_ids: + query = query.where(Conjugation.id.in_(conj_ids)) + + conjugations = session.execute(query).scalars().all() + + # Filter out blocked conjugation relationships + conjugations = [c for c in conjugations if (c.seq, c.from_seq) not in BLOCKED_CONJUGATIONS] + + if not conjugations: + _CONJ_DATA_CACHE[cache_key] = [] + return [] + + conj_ids_list = [c.id for c in conjugations] + + # Batch-fetch all source readings in one query + src_query = select(ConjSourceReading).where( + ConjSourceReading.conj_id.in_(conj_ids_list) + ) + if texts: + src_query = src_query.where(ConjSourceReading.text.in_(texts)) + + all_src_readings = session.execute(src_query).scalars().all() + src_by_conj: dict[int, list] = {} + for sr in all_src_readings: + src_by_conj.setdefault(sr.conj_id, []).append((sr.text, sr.source_text)) + + # Batch-fetch all conjugation properties in one query + all_props = session.execute( + select(ConjProp).where(ConjProp.conj_id.in_(conj_ids_list)) + ).scalars().all() + props_by_conj: dict[int, list] = {} + for prop in all_props: + props_by_conj.setdefault(prop.conj_id, []).append(prop) + + result = [] + for conj in conjugations: + src_map = src_by_conj.get(conj.id, []) + + if texts and not src_map: + continue + + for prop in props_by_conj.get(conj.id, []): + result.append(ConjData( + seq=conj.seq, + from_seq=conj.from_seq, + via=conj.via, + prop=prop, + src_map=src_map, + )) + + # Cache result (LRUCache handles eviction automatically) + _CONJ_DATA_CACHE[cache_key] = result + + return result + + +def get_word_conj_data( + session: Session, + word: Union[WordMatch, CompoundWord], +) -> List[ConjData]: + """ + Get conjugation data for a word match. + + Ports ichiran's word-conj-data method from dict.lisp. + + For simple words, gets conjugation data from the word's seq. + For compound words, gets conjugation data from the last word. + + Args: + session: Database session + word: WordMatch or CompoundWord + + Returns: + List of ConjData objects + """ + if isinstance(word, CompoundWord): + # For compound words, get conj data from last word + if word.words: + return get_word_conj_data(session, word.words[-1]) + return [] + + # For simple WordMatch + seq = word.seq + conj_ids = word.conjugations if word.conjugations and word.conjugations != 'root' else None + + if isinstance(conj_ids, list): + return get_conj_data(session, seq, conj_ids=conj_ids, texts=[word.text]) + elif not word.is_root: + return get_conj_data(session, seq, texts=[word.text]) + else: + return [] + + +def get_conj_type_name( + session: Session, + word: Union[WordMatch, CompoundWord], +) -> Optional[str]: + """ + Get human-readable conjugation type name for a word. + + Looks up the conjugation data for the word and returns the + human-readable name from CONJ_TYPE_NAMES mapping. + + Args: + session: Database session + word: WordMatch or CompoundWord + + Returns: + Human-readable conjugation type name (e.g., "Past (~ta)", "Conjunctive (~te)") + or None if no conjugation data found + """ + conj_data = get_word_conj_data(session, word) + if not conj_data: + return None + + # Get the first conjugation data entry + cd = conj_data[0] + if cd.prop and cd.prop.conj_type: + return CONJ_TYPE_NAMES.get(cd.prop.conj_type) + + return None + + +def get_conj_neg( + session: Session, + word: Union[WordMatch, CompoundWord], +) -> bool: + """ + Get whether a word is in negative form. + + Args: + session: Database session + word: WordMatch or CompoundWord + + Returns: + True if the word is in negative form, False otherwise + """ + conj_data = get_word_conj_data(session, word) + if not conj_data: + return False + + cd = conj_data[0] + if cd.prop: + return bool(cd.prop.neg) + + return False + + +def get_conj_fml( + session: Session, + word: Union[WordMatch, CompoundWord], +) -> bool: + """ + Get whether a word is in formal/polite form. + + Args: + session: Database session + word: WordMatch or CompoundWord + + Returns: + True if the word is in formal form, False otherwise + """ + conj_data = get_word_conj_data(session, word) + if not conj_data: + return False + + cd = conj_data[0] + if cd.prop: + return bool(cd.prop.fml) + + return False + + +def get_source_text( + session: Session, + word: Union[WordMatch, CompoundWord], +) -> Optional[str]: + """ + Get source text (dictionary form) for a conjugated word. + + Looks up the conjugation data and finds the matching src_map entry + to return the source text (e.g., "だ" for "で", "食べる" for "食べた"). + + Args: + session: Database session + word: WordMatch or CompoundWord + + Returns: + Source text (dictionary form) or None if not found + """ + conj_data = get_word_conj_data(session, word) + if not conj_data: + return None + + # Get the word's text to match against src_map + word_text = word.text + + # Search through all conjugation data entries + for cd in conj_data: + if cd.src_map: + for text, src_text in cd.src_map: + if text == word_text: + return src_text + + return None diff --git a/himotoki/lookup/constants.py b/himotoki/lookup/constants.py new file mode 100644 index 0000000..f47997a --- /dev/null +++ b/himotoki/lookup/constants.py @@ -0,0 +1,27 @@ +""" +Lookup constants (non-scoring). +""" + +from typing import Dict, Set + +# Parts of speech that can be conjugated +POS_WITH_CONJ_RULES = frozenset([ + 'v1', 'v1s', 'v5aru', 'v5b', 'v5g', 'v5k', 'v5k-s', 'v5m', 'v5n', + 'v5r', 'v5r-i', 'v5s', 'v5t', 'v5u', 'v5u-s', 'v5uru', 'vk', 'vs', + 'vs-i', 'vs-s', 'vz', 'adj-i', 'adj-ix', 'adj-na', 'adj-no', +]) + +# Expression entries that are valid dictionary items but too coarse for our +# segmentation goals. We prefer compositional analysis for these. +SUPPRESS_SINGLE_TOKEN_SEQS: Set[int] = { + 2825978, # でしょうか -> prefer でしょう + か +} + +# Import conjugation constants from central location +from himotoki.constants import ( + CONJ_ADVERBIAL, CONJ_ADJECTIVE_STEM, CONJ_NEGATIVE_STEM, + CONJ_CAUSATIVE_SU, CONJ_ADJECTIVE_LITERARY, + CONJ_TYPE_NAMES, WEAK_CONJ_FORMS, SKIP_CONJ_FORMS, + CONJ_VOLITIONAL, CONJ_TE, CONJ_POTENTIAL, CONJ_IMPERATIVE, +) + diff --git a/himotoki/lookup/find_word.py b/himotoki/lookup/find_word.py new file mode 100644 index 0000000..5177142 --- /dev/null +++ b/himotoki/lookup/find_word.py @@ -0,0 +1,196 @@ +""" +Database word lookup functions. +""" + +from typing import Optional, List, Set, Union + +from sqlalchemy import select, and_ +from sqlalchemy.orm import Session + +from himotoki.db.models import Entry, KanjiText, KanaText +from himotoki.characters import is_kana, as_hiragana +from himotoki.types import WordMatch, ConjData, CompoundWord +from himotoki.scoring.calc_score import MAX_WORD_LENGTH +from himotoki.lookup.constants import SUPPRESS_SINGLE_TOKEN_SEQS +from himotoki.lookup.conj_data import get_word_conj_data +from himotoki.scoring.caches import _WORD_CACHE + +def find_word( + session: Session, + word: str, + root_only: bool = False, +) -> List[WordMatch]: + """ + Find words matching the given text in the database. + + Args: + session: Database session + word: Text to search for + root_only: If True, only return root entries (not conjugations) + + Returns: + List of WordMatch objects + """ + # Include DB path so caches from different databases (e.g. test fixtures) + # do not collide. Session id alone is insufficient across connection pools. + bind = session.get_bind() + db_id = getattr(getattr(bind, "url", None), "database", None) or id(bind) + cache_key = (db_id, word, root_only) + if cache_key in _WORD_CACHE: + return _WORD_CACHE[cache_key] + + if len(word) > MAX_WORD_LENGTH: + return [] + + # Determine which table to search based on word content + if is_kana(word): + table = KanaText + else: + table = KanjiText + + if root_only: + # Join with entry to filter by root_p + query = ( + select(table) + .join(Entry, table.seq == Entry.seq) + .where(and_(table.text == word, Entry.root_p == True)) + ) + else: + query = select(table).where(table.text == word) + + results = session.execute(query).scalars().all() + results = [r for r in results if r.seq not in SUPPRESS_SINGLE_TOKEN_SEQS] + matches = [WordMatch(reading=r) for r in results] + _WORD_CACHE[cache_key] = matches + return matches + + +def find_word_as_hiragana( + session: Session, + word: str, + exclude_seqs: Optional[Set[int]] = None, +) -> List[WordMatch]: + """ + Find words by converting katakana to hiragana and searching. + + Args: + session: Database session + word: Text to search for (may contain katakana) + exclude_seqs: Seqs to exclude from results + + Returns: + List of WordMatch objects with proxy text + """ + hiragana = as_hiragana(word) + if hiragana == word: + return [] + + words = find_word(session, hiragana, root_only=True) + + if exclude_seqs: + words = [w for w in words if w.seq not in exclude_seqs] + + # TODO: Create proxy text objects that preserve original form + return words + + +def find_word_full( + session: Session, + word: str, + as_hiragana_lookup: bool = False, + counter: Union[bool, int] = False, +) -> List[WordMatch]: + """ + Full word lookup with multiple strategies. + + Args: + session: Database session + word: Text to search for + as_hiragana_lookup: If True, also try converting katakana to hiragana + counter: If True, look for counter patterns + + Returns: + List of WordMatch objects + """ + simple_words = find_word(session, word) + results = list(simple_words) + + # Add suffix lookup (find_word_suffix) + from himotoki.suffixes import find_word_suffix + suffix_words = find_word_suffix(session, word, matches=simple_words) + results.extend(suffix_words) + + if as_hiragana_lookup: + exclude = {w.seq for w in simple_words} + results.extend(find_word_as_hiragana(session, word, exclude)) + + # TODO: Add counter lookup + + return results + + +# ============================================================================ +# Conjugation-based Lookup Functions +# ============================================================================ + +def find_word_with_conj_prop( + session: Session, + word: str, + filter_fn: callable, + allow_root: bool = False, +) -> List[WordMatch]: + """ + Find words matching text with conjugation property filter. + + Ports ichiran's find-word-with-conj-prop from dict-grammar.lisp. + + Args: + session: Database session + word: Text to search for + filter_fn: Function to filter ConjData objects + allow_root: If True, also return root forms + + Returns: + List of WordMatch objects with conjugation IDs set + """ + results = [] + for match in find_word_full(session, word): + conj_data = get_word_conj_data(session, match) + conj_data_filtered = [cd for cd in conj_data if filter_fn(cd)] + # Use conj_id (foreign key to Conjugation.id), not prop.id + # This matches ichiran's (conj-id (conj-data-prop cdata)) + conj_ids = [cd.prop.conj_id if cd.prop else None for cd in conj_data_filtered] + conj_ids = [cid for cid in conj_ids if cid is not None] + + if conj_data_filtered or (not conj_data and allow_root): + match.conjugations = conj_ids if conj_ids else None + results.append(match) + + return results + + +def find_word_with_conj_type( + session: Session, + word: str, + *conj_types: int, +) -> List[WordMatch]: + """ + Find words matching text with specific conjugation types. + + Ports ichiran's find-word-with-conj-type from dict-grammar.lisp. + + Args: + session: Database session + word: Text to search for + *conj_types: Conjugation type IDs to match + + Returns: + List of WordMatch objects + """ + def filter_fn(cdata: ConjData) -> bool: + if cdata.prop and hasattr(cdata.prop, 'conj_type'): + return cdata.prop.conj_type in conj_types + return False + + return find_word_with_conj_prop(session, word, filter_fn) + diff --git a/himotoki/output.py b/himotoki/output.py deleted file mode 100644 index 0332963..0000000 --- a/himotoki/output.py +++ /dev/null @@ -1,2274 +0,0 @@ -""" -Output formatting module for himotoki. -Ports ichiran's word-info and JSON output functionality. - -This module provides: -- WordInfo: Data class for word information output -- get_senses_json: Generate sense/gloss JSON output -- word_info_gloss_json: Generate complete word info JSON -- dict_segment: Main entry point for segmentation with output -""" - -from dataclasses import dataclass, field -from typing import Optional, List, Dict, Any, Union, Tuple -from enum import Enum -import json - -from sqlalchemy import select, and_, func -from sqlalchemy.orm import Session - -from himotoki.db.models import ( - Entry, KanjiText, KanaText, Sense, Gloss, SenseProp, - Conjugation, ConjProp, ConjSourceReading, -) -from himotoki.raw_types import RawKanaReading, RawKanjiReading -from himotoki.lookup import ( - Segment, SegmentList, WordMatch, ConjData, - get_conj_data, find_word, - get_conj_type_name, get_conj_neg, get_conj_fml, get_source_text, - POS_WITH_CONJ_RULES, -) -from himotoki.constants import ( - CONJ_TYPE_NAMES, CONJ_STEP_GLOSSES, - CONJ_CAUSATIVE, CONJ_CAUSATIVE_PASSIVE, - get_conj_description, -) -from himotoki.suffixes import get_suffix_description - - -# ============================================================================ -# Enums and Constants -# ============================================================================ - -class WordType(Enum): - """Word type classification.""" - KANJI = 'kanji' - KANA = 'kana' - GAP = 'gap' - - -# Special conjugation info for entries that are standalone but represent -# conjugated forms of other entries (like です being formal non-past of だ) -# Format: seq -> (from_seq, conj_type, pos, neg, fml) -SPECIAL_CONJ_INFO: Dict[int, tuple] = { - 1628500: (2089020, 1, 'cop', False, True), # です = non-past formal of だ -} - -# Particle seqs that should NOT display conjugation info even if they match -# conjugated forms. These are standalone particles that happen to look like -# conjugated forms but should be treated as particles in most contexts. -# Example: で (seq 2028980) can be copula だ conjunctive form, but when it's -# matched as a standalone word, it's almost always the case particle. -SUPPRESS_CONJ_FOR_PARTICLES: set = { - 2028980, # で - case particle (location/means), not copula て-form -} - -# Words that have their own standalone dictionary meaning and should NOT -# display as conjugated forms of other verbs, even though the DB links them. -# These are typically nominalized verb forms that became independent nouns. -SUPPRESS_CONJ_FOR_NOUNS: set = { - 1382980, # 積もり (つもり) - intention/plan noun, not continuative of 積もる -} - -# Verbs that have their own dictionary entry but might be mistakenly identified -# as potential or other conjugated forms of different verbs. These should show -# their own glosses, not conjugation info from another verb. -SUPPRESS_CONJ_FOR_VERBS: set = { - 1345930, # 傷つける - standalone transitive v1, not potential of 傷つく -} - - -# ============================================================================ -# Data Classes -# ============================================================================ - -@dataclass -class WordInfo: - """ - Word information for output. - Mirrors ichiran's word-info class. - """ - type: WordType # :kanji, :kana, or :gap - text: str # Surface text - kana: Union[str, List[str]] # Reading(s) - - # Optional fields - true_text: Optional[str] = None # Original text (for proxy text) - seq: Optional[Union[int, List[int]]] = None # JMdict sequence number(s) - conjugations: Optional[Union[List[int], str]] = None # Conjugation IDs or 'root' - score: int = 0 - components: List['WordInfo'] = field(default_factory=list) # For compound words (WordInfo objects) - compound_texts: List[str] = field(default_factory=list) # Component texts for suffix compounds - alternative: bool = False # True if multiple readings available - primary: bool = True # Is this the primary reading - start: Optional[int] = None - end: Optional[int] = None - counter: Optional[List[Any]] = None # [value, ordinal] for counter words - skipped: int = 0 # Number of skipped alternatives - - # Conjugation info fields - is_compound: bool = False # True if this is a compound word - conj_type: Optional[str] = None # Human-readable conjugation type - conj_neg: bool = False # True if negative form - conj_fml: bool = False # True if formal/polite form - source_text: Optional[str] = None # Dictionary form for conjugated words - - # NEW: Meanings and POS - populated during analysis - meanings: List[str] = field(default_factory=list) # List of gloss strings - pos: Optional[str] = None # Part of speech (e.g., "[n,vs,vt]") - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary (for JSON serialization).""" - return { - 'type': self.type.value.upper(), - 'text': self.text, - 'truetext': self.true_text, - 'kana': self.kana, - 'seq': self.seq, - 'conjugations': 'ROOT' if self.conjugations == 'root' else self.conjugations, - 'score': self.score, - 'components': [c.to_dict() for c in self.components] if self.components else [], - 'compound_texts': self.compound_texts, - 'alternative': self.alternative, - 'primary': self.primary, - 'start': self.start, - 'end': self.end, - 'counter': self.counter, - 'skipped': self.skipped, - 'is_compound': self.is_compound, - 'conj_type': self.conj_type, - 'conj_neg': self.conj_neg, - 'conj_fml': self.conj_fml, - 'source_text': self.source_text, - 'meanings': self.meanings, - 'pos': self.pos, - } - - -@dataclass -class ConjStep: - """One step in a conjugation breakdown chain.""" - conj_type: str # e.g., "Passive", "Past (~ta)" - suffix: str # e.g., "られる", "た" - gloss: str # e.g., "is done (to)", "did/was" - neg: bool = False # True if this step is negative - fml: bool = False # True if this step is formal/polite - - -# ============================================================================ -# Reading Formatting -# ============================================================================ - -def reading_str(kanji: Optional[str], kana: str) -> str: - """ - Format reading as 'kanji 【kana】' or just 'kana'. - - Args: - kanji: Kanji text (may be None) - kana: Kana reading - - Returns: - Formatted reading string - """ - if kanji: - return f"{kanji} 【{kana}】" - return kana - - -def get_entry_reading(session: Session, seq: int) -> str: - """Get formatted reading for an entry by seq.""" - kanji_text = session.execute( - select(KanjiText.text) - .where(and_(KanjiText.seq == seq, KanjiText.ord == 0)) - ).scalars().first() - - kana_text = session.execute( - select(KanaText.text) - .where(and_(KanaText.seq == seq, KanaText.ord == 0)) - ).scalars().first() - - return reading_str(kanji_text, kana_text or '') - - -def get_kana_for_entry(session: Session, seq: int) -> str: - """Get the primary kana reading for an entry by seq.""" - kana_text = session.execute( - select(KanaText.text) - .where(and_(KanaText.seq == seq, KanaText.ord == 0)) - ).scalars().first() - - return kana_text or '' - - -def get_matching_kana_for_kanji(session: Session, seq: int, kanji_text: str) -> str: - """ - Get kana reading that best matches the kanji text's suffix pattern. - - For conjugated forms, the DB may have multiple kana readings with different - suffixes (e.g., いわないで and いわなくて for 言わないで). We want the kana - whose suffix matches the kanji's suffix. - - Args: - session: Database session - seq: Entry sequence number - kanji_text: The matched kanji text (e.g., "言わないで") - - Returns: - The best matching kana reading - """ - # Get all kana readings for this entry - kana_results = session.execute( - select(KanaText.text, KanaText.ord) - .where(KanaText.seq == seq) - .order_by(KanaText.ord) - ).all() - - if not kana_results: - return '' - - if len(kana_results) == 1: - return kana_results[0][0] - - # For multiple readings, try to match suffix patterns - # Common confusing pairs: ないで/なくて, ないと/なくと - # Extract the kana suffix from kanji (last few hiragana chars) - kanji_suffix = '' - for char in reversed(kanji_text): - code = ord(char) - # Hiragana: 0x3040-0x309F - if 0x3040 <= code <= 0x309F: - kanji_suffix = char + kanji_suffix - else: - break - - if kanji_suffix: - # Find a kana reading that ends with the same suffix - for kana, ord_num in kana_results: - if kana.endswith(kanji_suffix): - return kana - - # Fall back to first reading if no suffix match - return kana_results[0][0] - - -# ============================================================================ -# Global Meanings Cache for Performance -# ============================================================================ - -# Global cache for meanings and POS data (persists across analyze() calls) -# Format: {seq: (meanings_list, pos_string)} -_MEANINGS_CACHE: Dict[int, tuple] = {} - -# Maximum cache size to prevent unbounded memory growth -_MEANINGS_CACHE_MAX_SIZE = 50000 - - -def get_cached_meanings(seq: int) -> Optional[tuple]: - """Get cached meanings for a seq if available.""" - return _MEANINGS_CACHE.get(seq) - - -def cache_meanings(seq: int, meanings: List[str], pos: Optional[str]) -> None: - """Cache meanings for a seq.""" - global _MEANINGS_CACHE - # Simple LRU-ish: if cache is too large, clear half of it - if len(_MEANINGS_CACHE) >= _MEANINGS_CACHE_MAX_SIZE: - # Keep the most recent half (dict preserves insertion order in Python 3.7+) - items = list(_MEANINGS_CACHE.items()) - _MEANINGS_CACHE = dict(items[len(items) // 2:]) - _MEANINGS_CACHE[seq] = (meanings, pos) - - -def clear_meanings_cache() -> None: - """Clear the global meanings cache (for testing).""" - global _MEANINGS_CACHE - _MEANINGS_CACHE = {} - - -# ============================================================================ -# Batch Preloading for Performance -# ============================================================================ - -class ReadingsCache: - """ - Cache for batch-loaded readings to avoid repeated DB queries. - - This significantly improves performance by loading all needed readings - in a single query instead of one query per word. - """ - - def __init__(self): - self.kanji_readings: Dict[int, str] = {} # seq -> primary kanji text - self.kana_readings: Dict[int, str] = {} # seq -> primary kana text - self._loaded = False - - def preload(self, session: Session, seqs: set) -> None: - """ - Batch load all kanji and kana readings for the given seqs. - - Args: - session: Database session - seqs: Set of seq numbers to load - """ - if not seqs: - return - - # Batch load kanji readings (ord=0 is primary) - kanji_results = session.execute( - select(KanjiText.seq, KanjiText.text) - .where(and_(KanjiText.seq.in_(seqs), KanjiText.ord == 0)) - ).all() - for seq, text in kanji_results: - self.kanji_readings[seq] = text - - # Batch load kana readings (ord=0 is primary) - kana_results = session.execute( - select(KanaText.seq, KanaText.text) - .where(and_(KanaText.seq.in_(seqs), KanaText.ord == 0)) - ).all() - for seq, text in kana_results: - self.kana_readings[seq] = text - - self._loaded = True - - def get_kanji(self, seq: int) -> Optional[str]: - """Get cached kanji reading for seq.""" - return self.kanji_readings.get(seq) - - def get_kana(self, seq: int) -> str: - """Get cached kana reading for seq.""" - return self.kana_readings.get(seq, '') - - def get_source_text(self, from_seq: int) -> Optional[str]: - """Get source text (dictionary form) for a conjugation source.""" - # Prefer kanji form, fall back to kana - kanji = self.kanji_readings.get(from_seq) - if kanji: - return kanji - return self.kana_readings.get(from_seq) - - -def collect_seqs_from_path(path: list) -> set: - """ - Collect all seq numbers needed from a path for batch preloading. - - This includes: - - Word seqs - - Conjugation from_seqs (for source_text lookup) - - Args: - path: List of SegmentLists or Segments - - Returns: - Set of seq numbers to preload - """ - from himotoki.lookup import CompoundWord - from himotoki.counters import CounterText - - seqs = set() - - for item in path: - if isinstance(item, SegmentList): - # Get seqs from all segments in the list - for segment in item.segments: - _collect_segment_seqs(segment, seqs) - elif isinstance(item, Segment): - _collect_segment_seqs(item, seqs) - - return seqs - - -def _collect_segment_seqs(segment: Segment, seqs: set) -> None: - """Helper to collect seqs from a single segment.""" - from himotoki.lookup import CompoundWord - from himotoki.counters import CounterText - - word = segment.word - - # Skip CounterText - they don't have entries - if isinstance(word, CounterText): - if word.seq: - seqs.add(word.seq) - return - - # Add word seq - if hasattr(word, 'seq') and word.seq: - seqs.add(word.seq) - - # For CompoundWords, add all component seqs - if isinstance(word, CompoundWord): - if word.primary and hasattr(word.primary, 'seq'): - seqs.add(word.primary.seq) - for w in word.words: - if hasattr(w, 'seq') and w.seq: - seqs.add(w.seq) - - # Add from_seqs from conjugation data - conj_data = segment.info.get('conj', []) if segment.info else [] - for cd in conj_data: - if cd.from_seq: - seqs.add(cd.from_seq) - - -def word_info_reading_str(word_info: WordInfo) -> str: - """Get formatted reading string for WordInfo. - - For compounds where する is absorbed, merges the kana - (e.g., 'べんきょう しています' → 'べんきょうしています'). - """ - if word_info.type == WordType.KANJI or word_info.counter: - kana = word_info.kana - if isinstance(kana, list): - kana = '/'.join(kana) - # Merge kana when する is absorbed into a noun compound - if word_info.is_compound and word_info.components and isinstance(kana, str): - kana = _merge_suru_kana(word_info, kana) - return reading_str(word_info.text, kana) - return reading_str(None, word_info.text) - - -def _merge_suru_kana(word_info: WordInfo, kana: str) -> str: - """Merge kana spaces when する is absorbed into a noun compound. - - When the first suffix component is する (or a conjugated form of する), - remove the space between the noun kana and the verb kana. - - 'べんきょう しています' → 'べんきょうしています' - 'べんきょう した' → 'べんきょうした' - """ - if not word_info.components or len(word_info.components) < 2: - return kana - - primary = word_info.components[0] - suffix = word_info.components[1] - - # Only absorb plain する (not させる, される, いたす, etc.) - # Check if primary has no conjugation (it's a noun) - if primary.conjugations and primary.conjugations != 'root': - return kana - - suffix_kana = suffix.kana if isinstance(suffix.kana, str) else (suffix.kana[0] if suffix.kana else suffix.text) - # する or conjugated forms start with し or す - if suffix_kana and (suffix_kana.startswith("し") or suffix_kana.startswith("す")): - # Verify it's actually する-based by checking the kana pattern - # Valid: する, して, した, します, しない, しました, etc. - # Invalid: しまう (different verb), すぐ (unrelated) - primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) - expected_prefix = primary_kana + " " + suffix_kana[0] - if primary_kana and kana.startswith(expected_prefix): - return kana.replace(primary_kana + " ", primary_kana, 1) - - return kana - - -def has_conjugable_pos(session: Session, seq: int) -> bool: - """ - Check if a word has a part-of-speech that can be conjugated. - - This is used to determine if a dictionary form should have conj_type="Non-past". - Only returns True for verbs and i-adjectives which have true conjugation forms. - - Args: - session: Database session - seq: Entry sequence number - - Returns: - True if the word has a conjugable POS (verb or i-adjective) - """ - if seq is None: - return False - - # Parts of speech that should show "Non-past" for dictionary forms - # This is more restrictive than POS_WITH_CONJ_RULES because we only want - # verbs and i-adjectives that have true conjugated forms. - # Excludes: - # - adj-na, adj-no (which use copula for conjugation) - # - vs (verbal nouns that take する - the noun part doesn't conjugate) - NONPAST_POS = frozenset([ - 'v1', 'v1-s', 'v1s', 'v5aru', 'v5b', 'v5g', 'v5k', 'v5k-s', 'v5m', 'v5n', - 'v5r', 'v5r-i', 'v5s', 'v5t', 'v5u', 'v5u-s', 'v5uru', 'vk', - 'vs-i', 'vs-s', 'vz', 'adj-i', 'adj-ix', - 'cop', 'cop-da', 'aux-v', # Copula and auxiliary verbs - ]) - - # Query for POS tags - pos_results = session.execute( - select(SenseProp.text) - .join(Sense, SenseProp.sense_id == Sense.id) - .where(and_( - Sense.seq == seq, - SenseProp.tag == 'pos', - )) - ).scalars().all() - - # Check if any POS is in the conjugable set - for pos in pos_results: - if pos in NONPAST_POS: - return True - - return False - - -# ============================================================================ -# Sense/Gloss Functions -# ============================================================================ - -def get_senses_raw(session: Session, seq: Union[int, List[int]]) -> List[Dict[str, Any]]: - """ - Get raw sense data for an entry. - - Args: - session: Database session - seq: Entry sequence number or list of sequence numbers (for compound words) - - Returns: - List of sense dicts with ord, gloss, and props - """ - tags = ['pos', 's_inf', 'stagk', 'stagr', 'field'] - - # Handle list of seqs (compound words) - use first seq for senses - if isinstance(seq, list): - if not seq: - return [] - seq = seq[0] - - # Get glosses grouped by sense - glosses_query = ( - select(Sense.ord, func.group_concat(Gloss.text, '; ')) - .join(Gloss, Gloss.sense_id == Sense.id, isouter=True) - .where(Sense.seq == seq) - .group_by(Sense.id) - .order_by(Sense.ord) - ) - glosses = session.execute(glosses_query).all() - - # Get properties - props_query = ( - select(Sense.ord, SenseProp.tag, SenseProp.text) - .join(SenseProp, SenseProp.sense_id == Sense.id) - .where(and_(Sense.seq == seq, SenseProp.tag.in_(tags))) - .order_by(Sense.ord, SenseProp.tag, SenseProp.ord) - ) - props = session.execute(props_query).all() - - # Build sense list - sense_list = [ - {'ord': ord_val, 'gloss': gloss or '', 'props': {}} - for ord_val, gloss in glosses - ] - - # Organize props by sense and tag - for sord, tag, text in props: - for sense in sense_list: - if sense['ord'] == sord: - if tag not in sense['props']: - sense['props'][tag] = [] - sense['props'][tag].append(text) - break - - return sense_list - - -def get_senses(session: Session, seq: Union[int, List[int]]) -> List[Dict[str, Any]]: - """ - Get senses formatted for output. - - Args: - session: Database session - seq: Entry sequence number or list (for compound words) - - Returns list of dicts with pos_str, gloss, and props. - """ - result = [] - for sense in get_senses_raw(session, seq): - props = sense['props'] - pos = props.get('pos', []) - pos_str = f"[{','.join(pos)}]" if pos else '[]' - result.append({ - 'pos': pos_str, - 'gloss': sense['gloss'], - 'props': props, - }) - return result - - -def get_root_seq(session: Session, seq: int) -> Optional[int]: - """Get the root (dictionary form) seq for a conjugated entry. - - Walks the conjugation table to find the from_seq (root entry). - Returns None if no conjugation data found. - """ - conjs = session.execute( - select(Conjugation).where(Conjugation.seq == seq) - ).scalars().all() - if conjs: - return conjs[0].from_seq - return None - - -def get_senses_str(session: Session, seq: Union[int, List[int]]) -> str: - """Get senses as formatted string. - - If the entry has no senses (e.g., a conjugation-only entry), - returns an empty string. - """ - lines = [] - rpos = '[]' - - for i, sense in enumerate(get_senses(session, seq), 1): - pos = sense['pos'] - if pos != '[]': - rpos = pos - - gloss = sense['gloss'] - props = sense['props'] - - info = props.get('s_inf', []) - rinf = '; '.join(info) if info else None - - fields = props.get('field', []) - rfield = ','.join(fields) if fields else None - - parts = [f"{i}. {rpos}"] - if rfield: - parts.append(f"{{{rfield}}}") - if rinf: - parts.append(f"《{rinf}》") - parts.append(gloss) - - lines.append(' '.join(parts)) - - return '\n'.join(lines) - - -def get_senses_json( - session: Session, - seq: int, - pos_list: Optional[List[str]] = None, - reading: Optional[Any] = None, -) -> List[Dict[str, Any]]: - """ - Get senses as JSON-compatible dicts. - - Args: - session: Database session - seq: Entry sequence number - pos_list: Filter to these POS tags - reading: Filter to senses matching this reading - - Returns: - List of sense dicts for JSON output - """ - result = [] - rpos = '[]' - - for sense in get_senses(session, seq): - pos = sense['pos'] - if pos != '[]': - rpos = pos - - # POS filtering - if pos_list: - lpos = pos[1:-1].split(',') if pos != '[]' else [] - if not any(p in pos_list for p in lpos): - continue - - gloss = sense['gloss'] - props = sense['props'] - - js = {'pos': rpos, 'gloss': gloss} - - # Add field info - fields = props.get('field', []) - if fields: - js['field'] = f"{{{','.join(fields)}}}" - - # Add sense info - info = props.get('s_inf', []) - if info: - js['info'] = '; '.join(info) - - result.append(js) - - return result - - -# ============================================================================ -# Conjugation Info Functions -# ============================================================================ - -# get_conj_description is now imported from himotoki.constants - - -def conj_prop_json(prop: ConjProp) -> Dict[str, Any]: - """Convert conjugation property to JSON dict.""" - js = { - 'pos': prop.pos, - 'type': get_conj_description(prop.conj_type), - } - if prop.neg: - js['neg'] = True - if prop.fml: - js['fml'] = True - return js - - -def conj_info_json( - session: Session, - seq: int, - conjugations: Optional[List[int]] = None, - text: Optional[str] = None, -) -> List[Dict[str, Any]]: - """ - Get conjugation info as JSON. - - Args: - session: Database session - seq: Entry sequence number - conjugations: Specific conjugation IDs to include - text: Filter by text - - Returns: - List of conjugation info dicts - """ - result = [] - - # Get conjugations - query = select(Conjugation).where(Conjugation.seq == seq) - if conjugations and conjugations != 'root': - query = query.where(Conjugation.id.in_(conjugations)) - - conjs = session.execute(query).scalars().all() - - for conj in conjs: - # Get properties - props = session.execute( - select(ConjProp).where(ConjProp.conj_id == conj.id) - ).scalars().all() - - if not props: - continue - - js = { - 'prop': [conj_prop_json(p) for p in props], - 'reading': get_entry_reading(session, conj.from_seq), - 'gloss': get_senses_json(session, conj.from_seq), - 'readok': True, - } - - result.append(js) - - # Check for special conj info for standalone entries that represent conjugated forms - if not result and seq in SPECIAL_CONJ_INFO: - from_seq, conj_type, pos, neg, fml = SPECIAL_CONJ_INFO[seq] - js = { - 'prop': [{ - 'pos': pos, - 'type': get_conj_description(conj_type), - 'neg': neg, - 'fml': fml, - }], - 'reading': get_entry_reading(session, from_seq), - 'gloss': get_senses_json(session, from_seq), - 'readok': True, - } - result.append(js) - - return result - - -# ============================================================================ -# WordInfo Creation Functions -# ============================================================================ - -def word_info_from_word_match( - session: Session, - word_match, - cache: Optional[ReadingsCache] = None, -) -> WordInfo: - """ - Create a simple WordInfo from a WordMatch object. - Used for creating component WordInfo objects in compound words. - - Args: - session: Database session - word_match: WordMatch object from suffix/split compounds - cache: Optional preloaded readings cache for performance - - Returns: - WordInfo object with basic info (text, kana, seq, conjugations) - """ - reading = word_match.reading - - # Determine word type and kana - if word_match.word_type == 'kanji': - word_type = WordType.KANJI - # Get kana from best_kana or matching function - kana = getattr(reading, 'best_kana', None) - if not kana: - kana = get_matching_kana_for_kanji(session, word_match.seq, reading.text) - if not kana: - # Last resort - use empty string - kana = '' - else: - word_type = WordType.KANA - kana = reading.text - - # Get conjugation info if available - conj_type_name = None - source_text = None - - # If there are conjugations, try to get the first one's info - if word_match.conjugations and word_match.conjugations != 'root': - conj_ids = word_match.conjugations if isinstance(word_match.conjugations, list) else [word_match.conjugations] - if conj_ids: - conj_id = conj_ids[0] - # Look up conjugation info - Conjugation table, then get props - from himotoki.db.models import Conjugation - conj = session.query(Conjugation).filter(Conjugation.id == conj_id).first() - if conj: - # Get conj_type from props relationship (not directly on Conjugation) - if conj.props: - prop = conj.props[0] - conj_type_name = CONJ_TYPE_NAMES.get(prop.conj_type) - # Get source text from from_seq - if conj.from_seq: - if cache: - source_text = cache.get_source_text(conj.from_seq) - else: - kanji_text = session.execute( - select(KanjiText.text) - .where(and_(KanjiText.seq == conj.from_seq, KanjiText.ord == 0)) - ).scalars().first() - source_text = kanji_text - else: - # No conjugation data - this is a dictionary form - # Set conj_type to "Non-past" for verbs and adjectives - if has_conjugable_pos(session, word_match.seq): - conj_type_name = "Non-past" - - return WordInfo( - type=word_type, - text=word_match.text, - kana=kana, - seq=word_match.seq, - conjugations=word_match.conjugations if word_match.conjugations != 'root' else None, - conj_type=conj_type_name, - source_text=source_text, - ) - - -def word_info_from_segment( - session: Session, - segment: Segment, - cache: Optional[ReadingsCache] = None, -) -> WordInfo: - """ - Create WordInfo from a segment. - - Args: - session: Database session - segment: Segment with word match - cache: Optional preloaded readings cache for performance - - Returns: - WordInfo object - """ - from himotoki.lookup import CompoundWord - from himotoki.counters import CounterText - - word = segment.word - - # Handle CounterText specially - if isinstance(word, CounterText): - word_type = WordType.KANJI # Counters are typically kanji - return WordInfo( - type=word_type, - text=segment.get_text(), - kana=word.kana, - true_text=word.text, - seq=word.seq, - conjugations=[], - score=int(segment.score), - start=segment.start, - end=segment.end, - ) - - # Handle CompoundWord specially - if isinstance(word, CompoundWord): - word_type = WordType.KANA if word.word_type == 'kana' else WordType.KANJI - - # Get conjugation data from segment.info (computed by calc_score) - # This is the authoritative source for conjugation info - conj_data = segment.info.get('conj', []) if segment.info else [] - - # Extract conjugation IDs from conj_data - # Use conj_id (foreign key to Conjugation.id), not prop.id - conjugations = None - if conj_data: - conj_ids = [cd.prop.conj_id if cd.prop else None for cd in conj_data] - conj_ids = [cid for cid in conj_ids if cid is not None] - conjugations = conj_ids if conj_ids else None - - # Extract conjugation info directly from conj_data - conj_type_name = None - conj_neg = False - conj_fml = False - source_text = None - - if conj_data: - # Get the first conjugation data entry (primary conjugation) - cd = conj_data[0] - if cd.prop: - # Get conj_type name from the mapping - conj_type_name = CONJ_TYPE_NAMES.get(cd.prop.conj_type) - conj_neg = bool(cd.prop.neg) - conj_fml = bool(cd.prop.fml) - - # Get source_text - prefer kanji form from from_seq, fall back to src_map - # This matches ichiran's behavior of showing the dictionary form - if cd.from_seq: - # Use cache if available, otherwise query DB - if cache: - source_text = cache.get_source_text(cd.from_seq) - else: - # Try to get kanji form first - kanji_text = session.execute( - select(KanjiText.text) - .where(and_(KanjiText.seq == cd.from_seq, KanjiText.ord == 0)) - ).scalars().first() - if kanji_text: - source_text = kanji_text - else: - # Fall back to kana form - kana_text = session.execute( - select(KanaText.text) - .where(and_(KanaText.seq == cd.from_seq, KanaText.ord == 0)) - ).scalars().first() - if kana_text: - source_text = kana_text - else: - # Fallback: get conjugation info from the CompoundWord itself - # This handles suffix-created compounds where segment.info doesn't have conj data - conj_info = word.get_conjugation_info(session) - if conj_info.get('conj_type') is not None: - conj_type_name = CONJ_TYPE_NAMES.get(conj_info['conj_type']) - conj_neg = bool(conj_info.get('neg', False)) - conj_fml = bool(conj_info.get('fml', False)) - source_text = conj_info.get('source_text') - - # Also get conjugation IDs from the last word - if word.conjugations: - conjugations = word.conjugations - - # Get component texts from CompoundWord.components property - # This returns the text of each word in the compound - compound_texts = word.components if word.components else [] - - # Create component WordInfo objects from the underlying WordMatch objects - # This provides detailed info (including POS) for each part of the compound - component_word_infos = [] - if hasattr(word, 'words') and word.words: - for wm in word.words: - try: - comp_wi = word_info_from_word_match(session, wm, cache) - component_word_infos.append(comp_wi) - except Exception: - # If we fail to create a component WordInfo, skip it - pass - - return WordInfo( - type=word_type, - text=segment.get_text(), - kana=word.kana, # Use compound's full kana - true_text=word.text, - seq=word.seq, # This is an int for compound words (primary's seq) - conjugations=conjugations, - score=int(segment.score), - components=component_word_infos, # NEW: Component WordInfo objects - start=segment.start, - end=segment.end, - is_compound=True, - compound_texts=compound_texts, - conj_type=conj_type_name, - conj_neg=conj_neg, - conj_fml=conj_fml, - source_text=source_text, - ) - - reading = word.reading - - # Determine kana reading - # Handle both ORM objects (KanjiText/KanaText) and raw namedtuples (RawKanjiReading/RawKanaReading) - if isinstance(reading, (KanjiText, RawKanjiReading)): - word_type = WordType.KANJI - # Get best kana for kanji text - try best_kana attr, then suffix matching - kana = reading.best_kana - if not kana: - # Use suffix matching to get correct kana (e.g., ないで vs なくて) - kanji_text = reading.text - kana = get_matching_kana_for_kanji(session, word.seq, kanji_text) - else: - word_type = WordType.KANA - kana = reading.text - - # Get conjugation data from segment.info (computed by calc_score) - # This is the authoritative source for conjugation info - conj_data = segment.info.get('conj', []) if segment.info else [] - - # Suppress conjugation info for standalone particles - # These words have their own dictionary meaning as particles, and - # showing conjugation info (e.g., で as copula て-form) is misleading - if word.seq in SUPPRESS_CONJ_FOR_PARTICLES: - conj_data = [] - - # Suppress conjugation info for nouns that have their own dictionary meaning - # These are nominalized verb forms that became independent nouns - if word.seq in SUPPRESS_CONJ_FOR_NOUNS: - conj_data = [] - - # Suppress conjugation info for verbs with their own dictionary entries - # These might look like conjugated forms but are standalone verbs - if word.seq in SUPPRESS_CONJ_FOR_VERBS: - conj_data = [] - - # Extract conjugation IDs for the conjugations field - conjugations = word.conjugations - if conjugations is None and conj_data: - # Use conj_id (foreign key to Conjugation.id), not prop.id - # This matches ichiran's (conj-id (conj-data-prop cdata)) - conj_ids = [cd.prop.conj_id if cd.prop else None for cd in conj_data] - conj_ids = [cid for cid in conj_ids if cid is not None] - conjugations = conj_ids if conj_ids else None - - # Extract conjugation info directly from conj_data - # This avoids re-querying the database and ensures we use the same data - conj_type_name = None - conj_neg = False - conj_fml = False - source_text = None - - if conj_data: - # Get the first conjugation data entry (primary conjugation) - cd = conj_data[0] - if cd.prop: - # Get conj_type name from the mapping - conj_type_name = CONJ_TYPE_NAMES.get(cd.prop.conj_type) - conj_neg = bool(cd.prop.neg) - conj_fml = bool(cd.prop.fml) - - # Get source_text - prefer kanji form from from_seq, fall back to src_map - # This matches ichiran's behavior of showing the dictionary form - if cd.from_seq: - # Use cache if available, otherwise query DB - if cache: - source_text = cache.get_source_text(cd.from_seq) - else: - # Try to get kanji form first - kanji_text = session.execute( - select(KanjiText.text) - .where(and_(KanjiText.seq == cd.from_seq, KanjiText.ord == 0)) - ).scalars().first() - if kanji_text: - source_text = kanji_text - else: - # Fall back to kana form - kana_text = session.execute( - select(KanaText.text) - .where(and_(KanaText.seq == cd.from_seq, KanaText.ord == 0)) - ).scalars().first() - if kana_text: - source_text = kana_text - else: - # No conjugation data - this is a dictionary form - # Set conj_type to "Non-past" for verbs and adjectives - if has_conjugable_pos(session, word.seq): - conj_type_name = "Non-past" - - return WordInfo( - type=word_type, - text=segment.get_text(), - kana=kana, - true_text=word.text, - seq=word.seq, - conjugations=conjugations, - score=int(segment.score), - start=segment.start, - end=segment.end, - is_compound=False, - conj_type=conj_type_name, - conj_neg=conj_neg, - conj_fml=conj_fml, - source_text=source_text, - ) - - -def word_info_from_segment_list( - session: Session, - segment_list: SegmentList, - cache: Optional[ReadingsCache] = None, -) -> WordInfo: - """ - Create WordInfo from a segment list (multiple interpretations). - - Args: - session: Database session - segment_list: SegmentList with multiple segments - cache: Optional preloaded readings cache for performance - - Returns: - WordInfo object (possibly with alternatives) - """ - segments = segment_list.segments - - if not segments: - return WordInfo( - type=WordType.GAP, - text='', - kana='', - start=segment_list.start, - end=segment_list.end, - ) - - # Create WordInfo for each segment (pass cache) - wi_list = [word_info_from_segment(session, seg, cache) for seg in segments] - wi1 = wi_list[0] - max_score = wi1.score - - # Filter out low-scoring alternatives - cutoff = max_score * 0.67 # SEGMENT_SCORE_CUTOFF - wi_list = [wi for wi in wi_list if wi.score >= cutoff] - - matches = segment_list.matches - - if len(wi_list) == 1: - wi1.skipped = matches - 1 - return wi1 - - # Multiple alternatives - kana_list = [] - seq_list = [] - for wi in wi_list: - if isinstance(wi.kana, list): - kana_list.extend(wi.kana) - else: - kana_list.append(wi.kana) - if wi.seq: - if isinstance(wi.seq, list): - seq_list.extend(wi.seq) - else: - seq_list.append(wi.seq) - - # Remove duplicates while preserving order - seen_kana = set() - unique_kana = [] - for k in kana_list: - if k not in seen_kana: - unique_kana.append(k) - seen_kana.add(k) - - return WordInfo( - type=wi1.type, - text=wi1.text, - kana=unique_kana if len(unique_kana) > 1 else (unique_kana[0] if unique_kana else ''), - seq=seq_list if len(seq_list) > 1 else (seq_list[0] if seq_list else None), - components=wi_list, - alternative=True, - score=wi1.score, - start=segment_list.start, - end=segment_list.end, - skipped=matches - len(wi_list), - # Propagate conjugation info from primary alternative - conjugations=wi1.conjugations, - conj_type=wi1.conj_type, - conj_neg=wi1.conj_neg, - conj_fml=wi1.conj_fml, - source_text=wi1.source_text, - ) - - -def word_info_from_text(text: str) -> WordInfo: - """Create gap WordInfo for unmatched text.""" - return WordInfo( - type=WordType.GAP, - text=text, - kana=text, - ) - - -# ============================================================================ -# WordInfo JSON Output -# ============================================================================ - -def word_info_gloss_json( - session: Session, - word_info: WordInfo, - root_only: bool = False, -) -> Dict[str, Any]: - """ - Generate JSON output for WordInfo. - - Args: - session: Database session - word_info: WordInfo to convert - root_only: If True, skip conjugation info - - Returns: - Dict ready for JSON serialization - """ - js = { - 'reading': word_info_reading_str(word_info), - 'text': word_info.text, - 'kana': word_info.kana, - } - - if word_info.score: - js['score'] = word_info.score - - if word_info.alternative: - # Multiple interpretations - js['alternative'] = [ - word_info_gloss_json(session, wi, root_only) - for wi in word_info.components - ] - return js - - if word_info.components: - # Compound word with component WordInfo objects - js['compound'] = [wi.text for wi in word_info.components] - js['components'] = [ - word_info_gloss_json(session, wi, root_only) - for wi in word_info.components - ] - return js - - # Handle compound words that don't have component WordInfo objects - # (e.g., from suffix-based compounds where we only have the text) - if word_info.is_compound: - seq = word_info.seq - if seq: - js['seq'] = seq - # Add gloss from the main word's seq - glosses = get_senses(session, seq) - if glosses: - js['gloss'] = glosses - - # Add compound texts if available (for ichiran compatibility) - if word_info.compound_texts: - js['compound'] = word_info.compound_texts - - # Build conjugation info directly from WordInfo fields - # (the conjugation data was already extracted in word_info_from_segment) - if word_info.conj_type: - conj_prop = { - 'type': word_info.conj_type, - } - if word_info.conj_neg: - conj_prop['neg'] = True - if word_info.conj_fml: - conj_prop['fml'] = True - - conj_entry = { - 'prop': [conj_prop], - 'readok': True, - } - - # Add source reading if available - if word_info.source_text: - conj_entry['reading'] = word_info.source_text - - js['conj'] = [conj_entry] - - return js - - if word_info.counter: - # Counter word - value, ordinal = word_info.counter - js['counter'] = {'value': value, 'ordinal': ordinal} - if word_info.seq: - js['seq'] = word_info.seq - gloss = get_senses_json(session, word_info.seq, pos_list=['ctr']) - if gloss: - js['gloss'] = gloss - return js - - # Regular word - seq = word_info.seq - if seq: - js['seq'] = seq - - if root_only or word_info.conjugations is None or word_info.conjugations == 'root': - gloss = get_senses_json(session, seq) - if gloss: - js['gloss'] = gloss - - # Get conjugation info - # Check for regular conjugations OR special conj info for standalone copulae - # Note: seq can be a list for compound words, so we need to check for hashability - has_conjugations = word_info.conjugations and word_info.conjugations != 'root' - has_special_conj = isinstance(seq, int) and seq in SPECIAL_CONJ_INFO - - if seq and (has_conjugations or has_special_conj): - conj = conj_info_json( - session, seq, - conjugations=word_info.conjugations if has_conjugations else None, - text=word_info.true_text, - ) - if conj: - js['conj'] = conj - - return js - - -# ============================================================================ -# Main Entry Points -# ============================================================================ - -def fill_segment_path( - session: Session, - text: str, - path: List[SegmentList], - include_meanings: bool = True, -) -> List[WordInfo]: - """ - Fill gaps in segment path and convert to WordInfo list. - - Args: - session: Database session - text: Original text - path: List of SegmentLists or Segments from find_best_path - include_meanings: If True, populate meanings field (default True) - - Returns: - List of WordInfo objects covering the entire text - """ - # Batch preload all readings for performance - # This reduces N queries to 2 queries (one for kanji, one for kana) - seqs = collect_seqs_from_path(path) - cache = ReadingsCache() - cache.preload(session, seqs) - - result = [] - idx = 0 - - for item in path: - if isinstance(item, SegmentList): - segment_list = item - # Add gap before this segment if needed - if segment_list.start > idx: - gap_text = text[idx:segment_list.start] - result.append(WordInfo( - type=WordType.GAP, - text=gap_text, - kana=gap_text, - start=idx, - end=segment_list.start, - )) - - # Add the segment (pass cache for optimized lookups) - wi = word_info_from_segment_list(session, segment_list, cache) - expanded = _split_copula_compound_for_output(wi) - result.extend(expanded) - idx = segment_list.end - elif isinstance(item, Segment): - segment = item - # Add gap before this segment if needed - if segment.start > idx: - gap_text = text[idx:segment.start] - result.append(WordInfo( - type=WordType.GAP, - text=gap_text, - kana=gap_text, - start=idx, - end=segment.start, - )) - - # Add the segment (pass cache for optimized lookups) - wi = word_info_from_segment(session, segment, cache) - expanded = _split_copula_compound_for_output(wi) - result.extend(expanded) - idx = segment.end - - # Add trailing gap if needed - if idx < len(text): - gap_text = text[idx:] - result.append(WordInfo( - type=WordType.GAP, - text=gap_text, - kana=gap_text, - start=idx, - end=len(text), - )) - - # Populate meanings if requested - if include_meanings: - populate_meanings(session, result) - - return result - - -def _split_copula_compound_for_output(wi: WordInfo) -> List[WordInfo]: - """Split na-adjective/noun + copula compounds for display output. - - Keeps internal parsing/scoring behavior unchanged while presenting - user-facing output as separate tokens (e.g., 綺麗 + です + ね). - """ - if not wi.is_compound or not wi.components or len(wi.components) != 2: - return [wi] - - primary, suffix = wi.components - suffix_kana = suffix.kana if isinstance(suffix.kana, str) else (suffix.kana[0] if suffix.kana else suffix.text) - if suffix_kana not in ('です', 'でした'): - return [wi] - - # Only split when primary itself is not conjugated (noun/na-adj style usage) - if primary.conjugations and primary.conjugations != 'root': - return [wi] - - return [primary, suffix] - - -def populate_meanings(session: Session, word_infos: List[WordInfo]) -> None: - """ - Populate meanings and pos fields for a list of WordInfo objects. - - Uses a global cache to avoid repeated DB queries for the same words. - Only queries the database for seqs not already in cache. - - Args: - session: Database session - word_infos: List of WordInfo objects to populate - """ - # Collect all seqs that need meanings, checking cache first - seq_to_words: Dict[int, List[WordInfo]] = {} - uncached_seqs: List[int] = [] - - for wi in word_infos: - if wi.seq and wi.type != WordType.GAP: - # Handle list seqs (compound words) - use first seq - seq = wi.seq[0] if isinstance(wi.seq, list) else wi.seq - if seq not in seq_to_words: - seq_to_words[seq] = [] - # Check if we already have this in cache - cached = get_cached_meanings(seq) - if cached is None: - uncached_seqs.append(seq) - seq_to_words[seq].append(wi) - - if not seq_to_words: - return - - # Build lookup dicts - start with empty, will populate from cache or DB - meanings_by_seq: Dict[int, List[str]] = {} - pos_by_seq: Dict[int, Optional[str]] = {} - - # Populate from cache for already-cached seqs - for seq in seq_to_words.keys(): - cached = get_cached_meanings(seq) - if cached is not None: - meanings_by_seq[seq] = cached[0] - pos_by_seq[seq] = cached[1] - - # Only query DB for uncached seqs - if uncached_seqs: - # Query all senses and glosses in one go - senses_query = ( - select(Sense.seq, Sense.ord, func.group_concat(Gloss.text, '; ')) - .join(Gloss, Gloss.sense_id == Sense.id, isouter=True) - .where(Sense.seq.in_(uncached_seqs)) - .group_by(Sense.id) - .order_by(Sense.seq, Sense.ord) - ) - senses_results = session.execute(senses_query).all() - - # Query POS for all uncached seqs - pos_query = ( - select(Sense.seq, SenseProp.text) - .join(SenseProp, SenseProp.sense_id == Sense.id) - .where(and_(Sense.seq.in_(uncached_seqs), SenseProp.tag == 'pos', Sense.ord == 0)) - .order_by(Sense.seq, SenseProp.ord) - ) - pos_results = session.execute(pos_query).all() - - # Build meanings by seq from DB results - for seq, ord_val, gloss in senses_results: - if seq not in meanings_by_seq: - meanings_by_seq[seq] = [] - if gloss: - meanings_by_seq[seq].append(gloss) - - # Build POS by seq from DB results - pos_by_seq_temp: Dict[int, List[str]] = {} - for seq, pos_text in pos_results: - if seq not in pos_by_seq_temp: - pos_by_seq_temp[seq] = [] - pos_by_seq_temp[seq].append(pos_text) - - for seq, tags in pos_by_seq_temp.items(): - pos_by_seq[seq] = f"[{','.join(tags)}]" - - # Cache the newly loaded data - for seq in uncached_seqs: - meanings = meanings_by_seq.get(seq, []) - pos = pos_by_seq.get(seq) - cache_meanings(seq, meanings, pos) - - # Populate WordInfo objects - for seq, word_list in seq_to_words.items(): - meanings = meanings_by_seq.get(seq, []) - pos = pos_by_seq.get(seq) - for wi in word_list: - wi.meanings = meanings - wi.pos = pos - - -def dict_segment( - session: Session, - text: str, - limit: int = 5, -) -> List[tuple]: - """ - Segment text and return WordInfo results. - - This is the main entry point, equivalent to ichiran's dict-segment. - - Args: - session: Database session - text: Text to segment - limit: Maximum number of segmentation results - - Returns: - List of (word_info_list, score) tuples - """ - from himotoki.segment import segment_text - - results = segment_text(session, text, limit=limit) - - return [ - (fill_segment_path(session, text, path), score) - for path, score in results - ] - - -def simple_segment(session: Session, text: str, limit: int = 5) -> List[WordInfo]: - """ - Simple segmentation returning just the best path. - - Args: - session: Database session - text: Text to segment - limit: Maximum paths to consider - - Returns: - List of WordInfo for the best segmentation - """ - results = dict_segment(session, text, limit=limit) - if results: - return results[0][0] - return [] - - -def segment_to_json( - session: Session, - text: str, - limit: int = 5, -) -> List[List[Any]]: - """ - Segment text and return ichiran-compatible JSON. - - This matches the output format of ichiran-cli -f. - - Args: - session: Database session - text: Text to segment - limit: Maximum segmentation results - - Returns: - JSON-compatible nested list structure - """ - from himotoki.characters import romanize_word - - results = dict_segment(session, text, limit=limit) - - output = [] - for word_infos, score in results: - segments = [] - for wi in word_infos: - # [romanized, {word_info_json}, []] - # The third element is for split info (not yet implemented) - romanized = romanize_word(wi.kana if isinstance(wi.kana, str) else wi.kana[0] if wi.kana else wi.text) - segment_json = word_info_gloss_json(session, wi) - segments.append([romanized, segment_json, []]) - - output.append([segments, score]) - - return output - - -def segment_to_text( - session: Session, - text: str, - limit: int = 1, -) -> str: - """ - Segment text and return formatted text output. - - This matches the output format of ichiran-cli -i. - - Args: - session: Database session - text: Text to segment - limit: Maximum segmentation results - - Returns: - Formatted text output - """ - from himotoki.characters import romanize_word - - results = dict_segment(session, text, limit=limit) - - if not results: - return text - - word_infos, score = results[0] - - lines = [] - - # Romanized line - romanized_parts = [] - for wi in word_infos: - kana = wi.kana if isinstance(wi.kana, str) else wi.kana[0] if wi.kana else wi.text - romanized_parts.append(romanize_word(kana)) - lines.append(' '.join(romanized_parts)) - - # Word info lines - for wi in word_infos: - if wi.type == WordType.GAP: - continue - - lines.append('') - - romanized = romanize_word(wi.kana if isinstance(wi.kana, str) else wi.kana[0] if wi.kana else wi.text) - lines.append(f"* {romanized} {word_info_reading_str(wi)}") - - if wi.seq: - senses = get_senses_str(session, wi.seq) - lines.append(senses) - - # Conjugation info (breakdown tree) - conj_strs = _get_conjugation_display(session, wi) - for cs in conj_strs: - lines.append(cs) - - return '\n'.join(lines) - - -def _get_conjugation_display( - session: Session, - wi: WordInfo, -) -> List[str]: - """Get conjugation display lines for a WordInfo. - - Handles: - - Single words with DB conjugation chains (via chain → tree) - - Multi-alternative words (seq is a list) - - Compound words (show chain for primary component + suffix components) - - Returns empty list if word is not conjugated or is a root form. - """ - if not wi.seq: - return [] - - # Compound words: show breakdown for each component that has conjugation - if wi.is_compound and wi.components: - return _get_compound_display(session, wi) - - # Multi-alternative words where alternatives are compounds - # (e.g., 食べられていた: alternative=True, components are compound WordInfos) - if wi.alternative and wi.components: - primary = wi.components[0] - if primary.is_compound and primary.components: - return _get_compound_display(session, primary) - - # Standalone alternatives may have conjugation data only on a - # non-primary alternative (e.g., だろう/でしょう/ではない). Fall back - # to the first alternative that carries conjugation metadata. - for alt in wi.components: - if alt.conjugations and alt.conjugations != 'root' and alt.seq: - alt_seq = alt.seq[0] if isinstance(alt.seq, list) else alt.seq - lines = format_conjugation_info(session, alt_seq, alt.conjugations) - if wi.text == 'ではない': - lines = [line.replace('(じゃない)', '(ではない)') for line in lines] - return lines - - conjugations = wi.conjugations - if not conjugations or conjugations == 'root': - return [] - - # For multi-alternative words (seq is a list), use the first seq - seq = wi.seq[0] if isinstance(wi.seq, list) else wi.seq - lines = format_conjugation_info(session, seq, conjugations) - if wi.text == 'ではない': - lines = [line.replace('(じゃない)', '(ではない)') for line in lines] - return lines - - -def _get_compound_display( - session: Session, - wi: WordInfo, -) -> List[str]: - """Get conjugation display for a compound word. - - Shows the primary component's conjugation chain, then lists - each suffix component on subsequent tree lines. - - Special handling: - - する is absorbed into a ← root line (勉強する instead of └─ する) - - いる/おる after て are hidden; て is relabeled as progressive - - Example for 勉強しています (4→2 levels): - ← 勉強する 【べんきょうする】 - └─ Conjunctive (~te, progressive) (て) - └─ Polite (ます) - - Example for 書かせられていた: - ← 書く【かく】 - └─ Causative-Passive (書かせられる): is made to do - └─ Conjunctive (~te, progressive) (て) - └─ Past (~ta) (た): did/was - """ - result = [] - - if not wi.components: - return result - - primary = wi.components[0] - - # Get the primary component's conjugation chain - primary_chain = [] - if primary.conjugations and primary.conjugations != 'root' and primary.seq: - primary_seq = primary.seq[0] if isinstance(primary.seq, list) else primary.seq - primary_chain = format_conjugation_info(session, primary_seq, primary.conjugations) - - if primary_chain: - result.extend(primary_chain) - - # Count existing tree depth from primary chain - depth = 0 - for line in primary_chain: - stripped = line.lstrip() - if stripped.startswith("└─"): - depth += 1 - - # Add remaining components as continuation of the tree - for comp in wi.components[1:]: - indent = " " * depth - comp_kana = comp.kana if isinstance(comp.kana, str) else (comp.kana[0] if comp.kana else comp.text) - # Fallback to text if kana is empty (e.g., abbreviation-based suffixes like もいい) - if not comp_kana: - comp_kana = comp.text - comp_seq = comp.seq[0] if isinstance(comp.seq, list) else comp.seq if comp.seq else None - - # Get suffix description for this component - desc = get_suffix_description(comp_seq, text=comp_kana) if (comp_seq or comp_kana) else None - desc_str = f" ({desc})" if desc else "" - - # na-adjective copula compounds (静かでした, 大丈夫です) - if not primary_chain and depth == 0 and comp_kana in ('です', 'でした'): - primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) - if primary.text and primary.text != primary_kana: - result.append(f" ← {primary.text}だ 【{primary_kana}だ】") - else: - result.append(f" ← {primary_kana}だ") - - result.append(" └─ Polite (です)") - depth = 1 - - if comp_kana == 'でした': - result.append(" └─ Past (~ta) (でした): did/was") - depth = 2 - continue - - # i-adj + すぎる compounds may arrive without explicit conjugation IDs - # on the primary component (e.g., 高すぎる). In that case, synthesize - # the source + adjective stem steps for readability. - if not primary_chain and depth == 0 and comp_kana == "すぎる": - primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) - if primary_kana and primary_kana.endswith("い"): - primary_seq = primary.seq[0] if isinstance(primary.seq, list) else primary.seq - if primary_seq: - result.append(f" ← {get_entry_reading(session, primary_seq)}") - stem_kana = primary_kana[:-1] - result.append(f" └─ Adjective Stem ({stem_kana}): stem") - depth = 1 - indent = " " * depth - - if comp.conjugations and comp.conjugations != 'root' and comp.seq: - # This suffix component is itself conjugated (e.g., いた = past of いる) - comp_chain = format_conjugation_info(session, comp_seq, comp.conjugations) - if comp_chain: - # Extract auxiliary verb name from root line - aux_name = _extract_aux_name_from_chain(comp_chain) - - if aux_name == "する" and not primary_chain: - # ABSORB する: show ← noun+する root line - primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) - if primary.text and primary.text != primary_kana: - result.append(f" ← {primary.text}する 【{primary_kana}する】") - else: - result.append(f" ← {primary_kana}する") - # Add する's conjugation children (skip the ← root line) - for line in comp_chain[1:]: - stripped = line.lstrip() - if stripped.startswith("└─"): - sub_indent = " " * depth - result.append(f" {sub_indent}{stripped}") - depth += 1 - - elif aux_name in ("いる", "おる") and _last_line_is_te(result): - # HIDE いる/おる: relabel て as progressive, promote children - _relabel_te_progressive(result) - # Add いる/おる's conjugation children (skip the ← root line) - for line in comp_chain[1:]: - stripped = line.lstrip() - if stripped.startswith("└─"): - sub_indent = " " * depth - result.append(f" {sub_indent}{stripped}") - depth += 1 - - else: - # Normal: merge sub-chain into tree - for line in comp_chain: - stripped = line.lstrip() - if stripped.startswith("←"): - sub_indent = " " * depth - result.append(f" {sub_indent}└─ {aux_name}{desc_str}") - depth += 1 - elif stripped.startswith("└─"): - sub_indent = " " * depth - result.append(f" {sub_indent}{stripped}") - depth += 1 - else: - result.append(f" {indent}└─ {comp_kana}{desc_str}") - depth += 1 - else: - # Unconjugated suffix component - if comp_kana == "する" and not primary_chain: - # ABSORB unconjugated する: show ← noun+する root line - primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) - if primary.text and primary.text != primary_kana: - result.append(f" ← {primary.text}する 【{primary_kana}する】") - else: - result.append(f" ← {primary_kana}する") - elif comp_kana in ("いる", "おる") and _last_line_is_te(result): - # HIDE unconjugated いる/おる: relabel て as progressive - _relabel_te_progressive(result) - else: - result.append(f" {indent}└─ {comp_kana}{desc_str}") - depth += 1 - - return result - - -def _extract_aux_name_from_chain(comp_chain: List[str]) -> Optional[str]: - """Extract the auxiliary verb kana name from a conjugation chain's root line. - - Given a chain like [" ← 為る 【する】", " └─ Past (~ta) (た)"], - returns "する" (the kana inside 【...】). - """ - if not comp_chain: - return None - first = comp_chain[0].lstrip() - if not first.startswith("←"): - return None - aux_name = first[2:].strip() - if "【" in aux_name and "】" in aux_name: - start = aux_name.index("【") + 1 - end = aux_name.index("】") - return aux_name[start:end] - return aux_name - - -def _last_line_is_te(result: List[str]) -> bool: - """Check if the last tree line contains Conjunctive (~te).""" - for line in reversed(result): - stripped = line.lstrip() - if stripped.startswith("└─") and "Conjunctive (~te" in stripped: - return True - if stripped.startswith("└─") or stripped.startswith("←"): - return False - return False - - -def _relabel_te_progressive(result: List[str]) -> None: - """Find the last Conjunctive (~te) line and relabel it as progressive.""" - for i in range(len(result) - 1, -1, -1): - if "Conjunctive (~te)" in result[i]: - result[i] = result[i].replace( - "Conjunctive (~te)", - "Conjunctive (~te, progressive)", - ) - return - - -def format_conjugation_info( - session: Session, - seq: int, - conjugations: List[int], -) -> List[str]: - """Format conjugation info as text lines with breakdown tree. - - For simple conjugations (no via chain), shows the classic bracket format - plus a single-step tree. For multi-step conjugations (via chains), - shows a full derivation tree with box-drawing characters. - """ - result = [] - - query = select(Conjugation).where(Conjugation.seq == seq) - if conjugations: - query = query.where(Conjugation.id.in_(conjugations)) - - conjs = session.execute(query).scalars().all() - - # Use only the first (primary) conjugation to avoid showing - # duplicate trees from archaic/variant analyses (e.g., 忘れる vs 忘る) - for conj in conjs[:1]: - props = session.execute( - select(ConjProp).where(ConjProp.conj_id == conj.id) - ).scalars().all() - - for prop in props[:1]: - # Build the full conjugation chain (innermost first) - steps = _build_conj_chain(session, conj, prop) - - if steps: - # Show tree breakdown - root_reading = get_entry_reading(session, conj.from_seq) - result.append(f" ← {root_reading}") - current_depth = 0 - for step in steps: - indent = " " * current_depth - if step.conj_type == "Causative-Passive": - # Split into two separate steps: Causative then Passive - suffix = step.suffix - caus_suffix = suffix - pass_suffix = "" - if "られ" in suffix: - idx = suffix.index("られ") - caus_suffix = suffix[:idx] - pass_suffix = suffix[idx:] - elif "され" in suffix: - idx = suffix.index("され") - caus_suffix = suffix[:idx + 1] # include さ - pass_suffix = suffix[idx + 1:] # れる - result.append(f" {indent}└─ Causative ({caus_suffix}): makes do") - current_depth += 1 - indent = " " * current_depth - if pass_suffix: - result.append(f" {indent}└─ Passive ({pass_suffix}): is done (to)") - else: - result.append(f" {indent}└─ Passive: is done (to)") - current_depth += 1 - elif step.fml: - # Show Polite as its own tree step - polite_morpheme = "です" if step.suffix.startswith("で") else "ます" - result.append(f" {indent}└─ Polite ({polite_morpheme})") - current_depth += 1 - # If there's an actual conjugation beyond plain polite non-past - if step.conj_type != "Non-past" or step.neg: - indent = " " * current_depth - # Extract post-masu suffix - # ました→した, ません→せん - suffix = step.suffix - if step.conj_type == "Volitional": - # Use abstract volitional morpheme よう - # (neutral, stackable on any verb) - suffix = "よう" - elif "ま" in suffix: - idx = suffix.index("ま") - suffix = suffix[idx + 1:] # strip ま, keep した/せん - if step.neg: - if step.conj_type == "Non-past": - # Polite negative: せん → Negative (せん) - result.append(f" {indent}└─ Negative ({suffix}): not") - current_depth += 1 - elif suffix.startswith("せん") and len(suffix) > 2: - # Polite neg + conjugation: せんでした - # → Negative (せん) + Past (でした) - conj_suffix = suffix[2:] - result.append(f" {indent}└─ Negative (せん): not") - current_depth += 1 - indent = " " * current_depth - gloss_str = f": {step.gloss}" if step.gloss else "" - result.append(f" {indent}└─ {step.conj_type} ({conj_suffix}){gloss_str}") - current_depth += 1 - else: - # Fallback for other formal neg patterns - neg_part, conj_part = _split_neg_suffix(suffix) - result.append(f" {indent}└─ Negative ({neg_part}): not") - current_depth += 1 - if conj_part: - indent = " " * current_depth - gloss_str = f": {step.gloss}" if step.gloss else "" - result.append(f" {indent}└─ {step.conj_type} ({conj_part}){gloss_str}") - current_depth += 1 - else: - label = step.conj_type - gloss_str = f": {step.gloss}" if step.gloss else "" - result.append(f" {indent}└─ {label} ({suffix}){gloss_str}") - current_depth += 1 - else: - type_label = step.conj_type - gloss = step.gloss - suffix = step.suffix - if type_label == "Potential": - # Show dual label only when form is ambiguous - # Godan potential starts with え-dan kana (け,せ,て,...) - # which is clearly just Potential. Only れ/られ forms - # (ichidan verbs) are ambiguous with Passive. - _GODAN_POTENTIAL = set("えけせてねへめげ") - first_char = suffix[0] if suffix else "" - if first_char not in _GODAN_POTENTIAL: - type_label = "Potential/Passive" - gloss = "can do / is done (to)" - if step.neg: - # Split negative into separate tree levels - if type_label == "Non-past": - # Pure negative: ない is the non-past neg form - result.append(f" {indent}└─ Negative ({suffix}): not") - current_depth += 1 - elif suffix.endswith("ない"): - # Type + Negative: type applied first, then negated - # e.g., けない → Potential (け) + Negative (ない) - conj_suffix = suffix[:-2] - gloss_str = f": {gloss}" if gloss else "" - result.append(f" {indent}└─ {type_label} ({conj_suffix}){gloss_str}") - current_depth += 1 - indent = " " * current_depth - result.append(f" {indent}└─ Negative (ない): not") - current_depth += 1 - else: - # Negative + conjugation: negated first, then ない conjugated - # e.g., なかった → Negative (ない) + Past (かった) - neg_part, conj_part = _split_neg_suffix(suffix) - result.append(f" {indent}└─ Negative ({neg_part}): not") - current_depth += 1 - if conj_part: - indent = " " * current_depth - gloss_str = f": {gloss}" if gloss else "" - result.append(f" {indent}└─ {type_label} ({conj_part}){gloss_str}") - current_depth += 1 - else: - gloss_str = f": {gloss}" if gloss else "" - result.append(f" {indent}└─ {type_label} ({suffix}){gloss_str}") - current_depth += 1 - else: - # Fallback: flat format for cases where chain can't be built - neg_str = ' Negative' if prop.neg else ' Affirmative' - fml_str = ' Formal' if prop.fml else ' Plain' - type_desc = get_conj_description(prop.conj_type) - result.append(f" ← [{prop.pos}] {type_desc}{neg_str}{fml_str}") - result.append(f" {get_entry_reading(session, conj.from_seq)}") - - return result - - -def _build_conj_chain( - session: Session, - conj: Conjugation, - outer_prop: ConjProp, -) -> List[ConjStep]: - """ - Walk a conjugation's via chain and build an ordered list of ConjSteps. - - The chain goes from the root outward: - e.g., for 食べられなかった (eat+passive+neg+past): - root: 食べる - step 1: Passive (られる) - from via chain - step 2: Past (neg=True) - from outer prop - - For 行きました (go+polite+past): - root: 行く - step 1: Past (た) - single step (fml=True) - - Negative is stored as a flag on ConjStep.neg; the renderer splits - it into a separate "Negative" tree node at display time. - """ - from himotoki.lookup import get_conj_data - - steps = [] - - if conj.via is not None: - # Multi-step conjugation: walk the via chain from inside out - _collect_via_steps(session, conj.via, conj.from_seq, steps) - - # Add the outermost conjugation step - type_name = CONJ_TYPE_NAMES.get(outer_prop.conj_type, f"Type {outer_prop.conj_type}") - gloss = CONJ_STEP_GLOSSES.get(outer_prop.conj_type, "") - - # Get the suffix text: difference between conjugated text and source text - suffix = _get_conj_suffix(session, conj, outer_prop) - - steps.append(ConjStep( - conj_type=type_name, - suffix=suffix, - gloss=gloss, - neg=bool(outer_prop.neg), - fml=bool(outer_prop.fml), - )) - - return steps - - -def _collect_via_steps( - session: Session, - via_seq: int, - from_seq: int, - steps: List[ConjStep], -) -> None: - """ - Recursively collect conjugation steps from a via chain. - - Walks from the innermost via to the outermost, appending steps in order. - """ - from himotoki.lookup import get_conj_data - - via_data = get_conj_data(session, via_seq, from_seq=from_seq) - if not via_data: - return - - cd = via_data[0] - - # If this via has its own via, recurse deeper first - if cd.via is not None: - _collect_via_steps(session, cd.via, cd.from_seq, steps) - - # Now add this step - if cd.prop: - type_name = CONJ_TYPE_NAMES.get(cd.prop.conj_type, f"Type {cd.prop.conj_type}") - gloss = CONJ_STEP_GLOSSES.get(cd.prop.conj_type, "") - - # Get suffix text from src_map - # Pick the reading that gives the shortest non-empty suffix - # to avoid variant kanji (e.g., 喰べ for 食べ) breaking extraction - # Exception: Causative types prefer longest suffix: - # - Causative: standard させる over dialectal さす - # - Causative-Passive: full かせられる splits cleanly into かせ + られる - prefer_long = cd.prop.conj_type in (CONJ_CAUSATIVE, CONJ_CAUSATIVE_PASSIVE) - suffix = "" - if cd.src_map: - best_suffix = None - for text, src in cd.src_map: - s = _extract_suffix(text, src) - if s and s != text: # real match (not fallback to full text) - if best_suffix is None: - best_suffix = s - elif prefer_long and len(s) > len(best_suffix): - best_suffix = s - elif not prefer_long and len(s) < len(best_suffix): - best_suffix = s - - if best_suffix is not None: - suffix = best_suffix - else: - # Fallback: use first reading - suffix = _extract_suffix(cd.src_map[0][0], cd.src_map[0][1]) - - steps.append(ConjStep( - conj_type=type_name, - suffix=suffix, - gloss=gloss, - neg=bool(cd.prop.neg), - fml=bool(cd.prop.fml), - )) - - -def _get_conj_suffix( - session: Session, - conj: Conjugation, - prop: ConjProp, -) -> str: - """ - Get the suffix text that represents this conjugation step. - - Uses ConjSourceReading to find the conjugated form, then extracts - the part that differs from the source. - """ - src_readings = session.execute( - select(ConjSourceReading).where(ConjSourceReading.conj_id == conj.id) - ).scalars().all() - - if not src_readings: - return "" - - # Pick the reading that gives the best suffix. - # Usually shortest non-empty to avoid variant kanji breaking extraction. - # Exception: Causative types prefer longest (standard form over dialectal). - prefer_long = prop.conj_type in (CONJ_CAUSATIVE, CONJ_CAUSATIVE_PASSIVE) - best_suffix = None - for sr in src_readings: - suffix = _extract_suffix(sr.text, sr.source_text) - if suffix and suffix != sr.text: # suffix != full text means we found a real match - if best_suffix is None: - best_suffix = suffix - elif prefer_long and len(suffix) > len(best_suffix): - best_suffix = suffix - elif not prefer_long and len(suffix) < len(best_suffix): - best_suffix = suffix - - if best_suffix is not None: - return best_suffix - - # Fallback: use first reading - return _extract_suffix(src_readings[0].text, src_readings[0].source_text) - - -def _split_neg_suffix(suffix: str) -> Tuple[str, str]: - """Split a negative conjugation suffix into (neg_part, conj_part). - - The negative marker ない conjugates like an i-adjective, so negative - suffixes contain the neg marker plus its own conjugation ending. - - Examples: - なかった → (ない, かった) # neg + past - まなかった → (まない, かった) # neg + past (godan) - くなかった → (くない, かった) # neg + past (i-adj) - なくて → (ない, くて) # neg + te - なければ → (ない, ければ) # neg + provisional - ないで → (ない, で) # neg + te (variant) - ない → (ない, "") # just negative - """ - # ない-conjugation endings (most specific first to avoid partial matches) - for ending in ("かったら", "かった", "ければ", "くて"): - if suffix.endswith(ending): - neg_part = suffix[:-len(ending)] + "い" - return neg_part, ending - - # ないで pattern (negative te-form variant) - if suffix.endswith("で") and len(suffix) >= 3: - potential_neg = suffix[:-1] - if potential_neg.endswith("ない"): - return potential_neg, "で" - - # No further conjugation (e.g., just ない) - return suffix, "" - - -def _extract_suffix(conj_text: str, src_text: str) -> str: - """ - Extract the changed suffix by comparing conjugated and source text. - - e.g., (食べられなかった, 食べられる) → なかった - (食べられる, 食べる) → られる (replace る with られる) - (行きました, 行く) → きました - """ - # Find common prefix - common_len = 0 - for i, (c1, c2) in enumerate(zip(conj_text, src_text)): - if c1 == c2: - common_len = i + 1 - else: - break - - suffix_part = conj_text[common_len:] - return suffix_part if suffix_part else conj_text diff --git a/himotoki/output/__init__.py b/himotoki/output/__init__.py new file mode 100644 index 0000000..8754e8d --- /dev/null +++ b/himotoki/output/__init__.py @@ -0,0 +1,52 @@ +""" +Output formatting: WordInfo, JSON, and text output. +""" + +from himotoki.output.types import * # noqa: F401,F403 +from himotoki.output.meanings import ( # noqa: F401 + reading_str, + get_entry_reading, + get_kana_for_entry, + get_matching_kana_for_kanji, + get_cached_meanings, + cache_meanings, + clear_meanings_cache, + ReadingsCache, + collect_seqs_from_path, + word_info_reading_str, + has_conjugable_pos, + get_senses_raw, + get_senses, + get_root_seq, + get_senses_str, + get_senses_json, + conj_prop_json, + conj_info_json, + populate_meanings, +) +from himotoki.output.word_info import ( # noqa: F401 + word_info_from_word_match, + word_info_from_segment, + word_info_from_segment_list, + word_info_from_text, + fill_segment_path, +) +from himotoki.output.conjugation_display import ( # noqa: F401 + format_conjugation_info, + _get_conjugation_display, + _get_compound_display, + _extract_suffix, + _build_conj_chain, + _collect_via_steps, + _get_conj_suffix, + _split_neg_suffix, +) +from himotoki.output.format import ( # noqa: F401 + word_info_gloss_json, + dict_segment, + simple_segment, + segment_to_json, + segment_to_text, +) +from himotoki.constants import get_conj_description # noqa: F401 + diff --git a/himotoki/output/conjugation_display.py b/himotoki/output/conjugation_display.py new file mode 100644 index 0000000..08bbe19 --- /dev/null +++ b/himotoki/output/conjugation_display.py @@ -0,0 +1,628 @@ +""" +Conjugation display and breakdown tree formatting. +""" + +from typing import Optional, List, Dict, Any, Tuple + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from himotoki.db.models import Conjugation, ConjProp, ConjSourceReading +from himotoki.lookup import get_conj_data, get_conj_type_name, get_conj_neg, get_conj_fml, get_source_text +from himotoki.constants import ( + CONJ_TYPE_NAMES, CONJ_STEP_GLOSSES, CONJ_CAUSATIVE, CONJ_CAUSATIVE_PASSIVE, + get_conj_description, +) +from himotoki.suffixes import get_suffix_description +from himotoki.output.meanings import get_entry_reading +from himotoki.output.types import WordInfo, ConjStep, SUPPRESS_CONJ_FOR_PARTICLES, SUPPRESS_CONJ_FOR_NOUNS, SUPPRESS_CONJ_FOR_VERBS + +def _get_conjugation_display( + session: Session, + wi: WordInfo, +) -> List[str]: + """Get conjugation display lines for a WordInfo. + + Handles: + - Single words with DB conjugation chains (via chain → tree) + - Multi-alternative words (seq is a list) + - Compound words (show chain for primary component + suffix components) + + Returns empty list if word is not conjugated or is a root form. + """ + if not wi.seq: + return [] + + # Compound words: show breakdown for each component that has conjugation + if wi.is_compound and wi.components: + return _get_compound_display(session, wi) + + # Multi-alternative words where alternatives are compounds + # (e.g., 食べられていた: alternative=True, components are compound WordInfos) + if wi.alternative and wi.components: + primary = wi.components[0] + if primary.is_compound and primary.components: + return _get_compound_display(session, primary) + + # Standalone alternatives may have conjugation data only on a + # non-primary alternative (e.g., だろう/でしょう/ではない). Fall back + # to the first alternative that carries conjugation metadata. + for alt in wi.components: + if alt.conjugations and alt.conjugations != 'root' and alt.seq: + alt_seq = alt.seq[0] if isinstance(alt.seq, list) else alt.seq + lines = format_conjugation_info(session, alt_seq, alt.conjugations) + if wi.text == 'ではない': + lines = [line.replace('(じゃない)', '(ではない)') for line in lines] + return lines + + conjugations = wi.conjugations + if not conjugations or conjugations == 'root': + return [] + + # For multi-alternative words (seq is a list), use the first seq + seq = wi.seq[0] if isinstance(wi.seq, list) else wi.seq + lines = format_conjugation_info(session, seq, conjugations) + if wi.text == 'ではない': + lines = [line.replace('(じゃない)', '(ではない)') for line in lines] + return lines + + +def _get_compound_display( + session: Session, + wi: WordInfo, +) -> List[str]: + """Get conjugation display for a compound word. + + Shows the primary component's conjugation chain, then lists + each suffix component on subsequent tree lines. + + Special handling: + - する is absorbed into a ← root line (勉強する instead of └─ する) + - いる/おる after て are hidden; て is relabeled as progressive + + Example for 勉強しています (4→2 levels): + ← 勉強する 【べんきょうする】 + └─ Conjunctive (~te, progressive) (て) + └─ Polite (ます) + + Example for 書かせられていた: + ← 書く【かく】 + └─ Causative-Passive (書かせられる): is made to do + └─ Conjunctive (~te, progressive) (て) + └─ Past (~ta) (た): did/was + """ + result = [] + + if not wi.components: + return result + + primary = wi.components[0] + + # Get the primary component's conjugation chain + primary_chain = [] + if primary.conjugations and primary.conjugations != 'root' and primary.seq: + primary_seq = primary.seq[0] if isinstance(primary.seq, list) else primary.seq + primary_chain = format_conjugation_info(session, primary_seq, primary.conjugations) + + if primary_chain: + result.extend(primary_chain) + + # Count existing tree depth from primary chain + depth = 0 + for line in primary_chain: + stripped = line.lstrip() + if stripped.startswith("└─"): + depth += 1 + + # Add remaining components as continuation of the tree + for comp in wi.components[1:]: + indent = " " * depth + comp_kana = comp.kana if isinstance(comp.kana, str) else (comp.kana[0] if comp.kana else comp.text) + # Fallback to text if kana is empty (e.g., abbreviation-based suffixes like もいい) + if not comp_kana: + comp_kana = comp.text + comp_seq = comp.seq[0] if isinstance(comp.seq, list) else comp.seq if comp.seq else None + + # Get suffix description for this component + desc = get_suffix_description(comp_seq, text=comp_kana) if (comp_seq or comp_kana) else None + desc_str = f" ({desc})" if desc else "" + + # na-adjective copula compounds (静かでした, 大丈夫です) + if not primary_chain and depth == 0 and comp_kana in ('です', 'でした'): + primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) + if primary.text and primary.text != primary_kana: + result.append(f" ← {primary.text}だ 【{primary_kana}だ】") + else: + result.append(f" ← {primary_kana}だ") + + result.append(" └─ Polite (です)") + depth = 1 + + if comp_kana == 'でした': + result.append(" └─ Past (~ta) (でした): did/was") + depth = 2 + continue + + # i-adj + すぎる compounds may arrive without explicit conjugation IDs + # on the primary component (e.g., 高すぎる). In that case, synthesize + # the source + adjective stem steps for readability. + if not primary_chain and depth == 0 and comp_kana == "すぎる": + primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) + if primary_kana and primary_kana.endswith("い"): + primary_seq = primary.seq[0] if isinstance(primary.seq, list) else primary.seq + if primary_seq: + result.append(f" ← {get_entry_reading(session, primary_seq)}") + stem_kana = primary_kana[:-1] + result.append(f" └─ Adjective Stem ({stem_kana}): stem") + depth = 1 + indent = " " * depth + + if comp.conjugations and comp.conjugations != 'root' and comp.seq: + # This suffix component is itself conjugated (e.g., いた = past of いる) + comp_chain = format_conjugation_info(session, comp_seq, comp.conjugations) + if comp_chain: + # Extract auxiliary verb name from root line + aux_name = _extract_aux_name_from_chain(comp_chain) + + if aux_name == "する" and not primary_chain: + # ABSORB する: show ← noun+する root line + primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) + if primary.text and primary.text != primary_kana: + result.append(f" ← {primary.text}する 【{primary_kana}する】") + else: + result.append(f" ← {primary_kana}する") + # Add する's conjugation children (skip the ← root line) + for line in comp_chain[1:]: + stripped = line.lstrip() + if stripped.startswith("└─"): + sub_indent = " " * depth + result.append(f" {sub_indent}{stripped}") + depth += 1 + + elif aux_name in ("いる", "おる") and _last_line_is_te(result): + # HIDE いる/おる: relabel て as progressive, promote children + _relabel_te_progressive(result) + # Add いる/おる's conjugation children (skip the ← root line) + for line in comp_chain[1:]: + stripped = line.lstrip() + if stripped.startswith("└─"): + sub_indent = " " * depth + result.append(f" {sub_indent}{stripped}") + depth += 1 + + else: + # Normal: merge sub-chain into tree + for line in comp_chain: + stripped = line.lstrip() + if stripped.startswith("←"): + sub_indent = " " * depth + result.append(f" {sub_indent}└─ {aux_name}{desc_str}") + depth += 1 + elif stripped.startswith("└─"): + sub_indent = " " * depth + result.append(f" {sub_indent}{stripped}") + depth += 1 + else: + result.append(f" {indent}└─ {comp_kana}{desc_str}") + depth += 1 + else: + # Unconjugated suffix component + if comp_kana == "する" and not primary_chain: + # ABSORB unconjugated する: show ← noun+する root line + primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) + if primary.text and primary.text != primary_kana: + result.append(f" ← {primary.text}する 【{primary_kana}する】") + else: + result.append(f" ← {primary_kana}する") + elif comp_kana in ("いる", "おる") and _last_line_is_te(result): + # HIDE unconjugated いる/おる: relabel て as progressive + _relabel_te_progressive(result) + else: + result.append(f" {indent}└─ {comp_kana}{desc_str}") + depth += 1 + + return result + + +def _extract_aux_name_from_chain(comp_chain: List[str]) -> Optional[str]: + """Extract the auxiliary verb kana name from a conjugation chain's root line. + + Given a chain like [" ← 為る 【する】", " └─ Past (~ta) (た)"], + returns "する" (the kana inside 【...】). + """ + if not comp_chain: + return None + first = comp_chain[0].lstrip() + if not first.startswith("←"): + return None + aux_name = first[2:].strip() + if "【" in aux_name and "】" in aux_name: + start = aux_name.index("【") + 1 + end = aux_name.index("】") + return aux_name[start:end] + return aux_name + + +def _last_line_is_te(result: List[str]) -> bool: + """Check if the last tree line contains Conjunctive (~te).""" + for line in reversed(result): + stripped = line.lstrip() + if stripped.startswith("└─") and "Conjunctive (~te" in stripped: + return True + if stripped.startswith("└─") or stripped.startswith("←"): + return False + return False + + +def _relabel_te_progressive(result: List[str]) -> None: + """Find the last Conjunctive (~te) line and relabel it as progressive.""" + for i in range(len(result) - 1, -1, -1): + if "Conjunctive (~te)" in result[i]: + result[i] = result[i].replace( + "Conjunctive (~te)", + "Conjunctive (~te, progressive)", + ) + return + + +def format_conjugation_info( + session: Session, + seq: int, + conjugations: List[int], +) -> List[str]: + """Format conjugation info as text lines with breakdown tree. + + For simple conjugations (no via chain), shows the classic bracket format + plus a single-step tree. For multi-step conjugations (via chains), + shows a full derivation tree with box-drawing characters. + """ + result = [] + + query = select(Conjugation).where(Conjugation.seq == seq) + if conjugations: + query = query.where(Conjugation.id.in_(conjugations)) + + conjs = session.execute(query).scalars().all() + + # Use only the first (primary) conjugation to avoid showing + # duplicate trees from archaic/variant analyses (e.g., 忘れる vs 忘る) + for conj in conjs[:1]: + props = session.execute( + select(ConjProp).where(ConjProp.conj_id == conj.id) + ).scalars().all() + + for prop in props[:1]: + # Build the full conjugation chain (innermost first) + steps = _build_conj_chain(session, conj, prop) + + if steps: + # Show tree breakdown + root_reading = get_entry_reading(session, conj.from_seq) + result.append(f" ← {root_reading}") + current_depth = 0 + for step in steps: + indent = " " * current_depth + if step.conj_type == "Causative-Passive": + # Split into two separate steps: Causative then Passive + suffix = step.suffix + caus_suffix = suffix + pass_suffix = "" + if "られ" in suffix: + idx = suffix.index("られ") + caus_suffix = suffix[:idx] + pass_suffix = suffix[idx:] + elif "され" in suffix: + idx = suffix.index("され") + caus_suffix = suffix[:idx + 1] # include さ + pass_suffix = suffix[idx + 1:] # れる + result.append(f" {indent}└─ Causative ({caus_suffix}): makes do") + current_depth += 1 + indent = " " * current_depth + if pass_suffix: + result.append(f" {indent}└─ Passive ({pass_suffix}): is done (to)") + else: + result.append(f" {indent}└─ Passive: is done (to)") + current_depth += 1 + elif step.fml: + # Show Polite as its own tree step + polite_morpheme = "です" if step.suffix.startswith("で") else "ます" + result.append(f" {indent}└─ Polite ({polite_morpheme})") + current_depth += 1 + # If there's an actual conjugation beyond plain polite non-past + if step.conj_type != "Non-past" or step.neg: + indent = " " * current_depth + # Extract post-masu suffix + # ました→した, ません→せん + suffix = step.suffix + if step.conj_type == "Volitional": + # Use abstract volitional morpheme よう + # (neutral, stackable on any verb) + suffix = "よう" + elif "ま" in suffix: + idx = suffix.index("ま") + suffix = suffix[idx + 1:] # strip ま, keep した/せん + if step.neg: + if step.conj_type == "Non-past": + # Polite negative: せん → Negative (せん) + result.append(f" {indent}└─ Negative ({suffix}): not") + current_depth += 1 + elif suffix.startswith("せん") and len(suffix) > 2: + # Polite neg + conjugation: せんでした + # → Negative (せん) + Past (でした) + conj_suffix = suffix[2:] + result.append(f" {indent}└─ Negative (せん): not") + current_depth += 1 + indent = " " * current_depth + gloss_str = f": {step.gloss}" if step.gloss else "" + result.append(f" {indent}└─ {step.conj_type} ({conj_suffix}){gloss_str}") + current_depth += 1 + else: + # Fallback for other formal neg patterns + neg_part, conj_part = _split_neg_suffix(suffix) + result.append(f" {indent}└─ Negative ({neg_part}): not") + current_depth += 1 + if conj_part: + indent = " " * current_depth + gloss_str = f": {step.gloss}" if step.gloss else "" + result.append(f" {indent}└─ {step.conj_type} ({conj_part}){gloss_str}") + current_depth += 1 + else: + label = step.conj_type + gloss_str = f": {step.gloss}" if step.gloss else "" + result.append(f" {indent}└─ {label} ({suffix}){gloss_str}") + current_depth += 1 + else: + type_label = step.conj_type + gloss = step.gloss + suffix = step.suffix + if type_label == "Potential": + # Show dual label only when form is ambiguous + # Godan potential starts with え-dan kana (け,せ,て,...) + # which is clearly just Potential. Only れ/られ forms + # (ichidan verbs) are ambiguous with Passive. + _GODAN_POTENTIAL = set("えけせてねへめげ") + first_char = suffix[0] if suffix else "" + if first_char not in _GODAN_POTENTIAL: + type_label = "Potential/Passive" + gloss = "can do / is done (to)" + if step.neg: + # Split negative into separate tree levels + if type_label == "Non-past": + # Pure negative: ない is the non-past neg form + result.append(f" {indent}└─ Negative ({suffix}): not") + current_depth += 1 + elif suffix.endswith("ない"): + # Type + Negative: type applied first, then negated + # e.g., けない → Potential (け) + Negative (ない) + conj_suffix = suffix[:-2] + gloss_str = f": {gloss}" if gloss else "" + result.append(f" {indent}└─ {type_label} ({conj_suffix}){gloss_str}") + current_depth += 1 + indent = " " * current_depth + result.append(f" {indent}└─ Negative (ない): not") + current_depth += 1 + else: + # Negative + conjugation: negated first, then ない conjugated + # e.g., なかった → Negative (ない) + Past (かった) + neg_part, conj_part = _split_neg_suffix(suffix) + result.append(f" {indent}└─ Negative ({neg_part}): not") + current_depth += 1 + if conj_part: + indent = " " * current_depth + gloss_str = f": {gloss}" if gloss else "" + result.append(f" {indent}└─ {type_label} ({conj_part}){gloss_str}") + current_depth += 1 + else: + gloss_str = f": {gloss}" if gloss else "" + result.append(f" {indent}└─ {type_label} ({suffix}){gloss_str}") + current_depth += 1 + else: + # Fallback: flat format for cases where chain can't be built + neg_str = ' Negative' if prop.neg else ' Affirmative' + fml_str = ' Formal' if prop.fml else ' Plain' + type_desc = get_conj_description(prop.conj_type) + result.append(f" ← [{prop.pos}] {type_desc}{neg_str}{fml_str}") + result.append(f" {get_entry_reading(session, conj.from_seq)}") + + return result + + +def _build_conj_chain( + session: Session, + conj: Conjugation, + outer_prop: ConjProp, +) -> List[ConjStep]: + """ + Walk a conjugation's via chain and build an ordered list of ConjSteps. + + The chain goes from the root outward: + e.g., for 食べられなかった (eat+passive+neg+past): + root: 食べる + step 1: Passive (られる) - from via chain + step 2: Past (neg=True) - from outer prop + + For 行きました (go+polite+past): + root: 行く + step 1: Past (た) - single step (fml=True) + + Negative is stored as a flag on ConjStep.neg; the renderer splits + it into a separate "Negative" tree node at display time. + """ + from himotoki.lookup import get_conj_data + + steps = [] + + if conj.via is not None: + # Multi-step conjugation: walk the via chain from inside out + _collect_via_steps(session, conj.via, conj.from_seq, steps) + + # Add the outermost conjugation step + type_name = CONJ_TYPE_NAMES.get(outer_prop.conj_type, f"Type {outer_prop.conj_type}") + gloss = CONJ_STEP_GLOSSES.get(outer_prop.conj_type, "") + + # Get the suffix text: difference between conjugated text and source text + suffix = _get_conj_suffix(session, conj, outer_prop) + + steps.append(ConjStep( + conj_type=type_name, + suffix=suffix, + gloss=gloss, + neg=bool(outer_prop.neg), + fml=bool(outer_prop.fml), + )) + + return steps + + +def _collect_via_steps( + session: Session, + via_seq: int, + from_seq: int, + steps: List[ConjStep], +) -> None: + """ + Recursively collect conjugation steps from a via chain. + + Walks from the innermost via to the outermost, appending steps in order. + """ + from himotoki.lookup import get_conj_data + + via_data = get_conj_data(session, via_seq, from_seq=from_seq) + if not via_data: + return + + cd = via_data[0] + + # If this via has its own via, recurse deeper first + if cd.via is not None: + _collect_via_steps(session, cd.via, cd.from_seq, steps) + + # Now add this step + if cd.prop: + type_name = CONJ_TYPE_NAMES.get(cd.prop.conj_type, f"Type {cd.prop.conj_type}") + gloss = CONJ_STEP_GLOSSES.get(cd.prop.conj_type, "") + + # Get suffix text from src_map + # Pick the reading that gives the shortest non-empty suffix + # to avoid variant kanji (e.g., 喰べ for 食べ) breaking extraction + # Exception: Causative types prefer longest suffix: + # - Causative: standard させる over dialectal さす + # - Causative-Passive: full かせられる splits cleanly into かせ + られる + prefer_long = cd.prop.conj_type in (CONJ_CAUSATIVE, CONJ_CAUSATIVE_PASSIVE) + suffix = "" + if cd.src_map: + best_suffix = None + for text, src in cd.src_map: + s = _extract_suffix(text, src) + if s and s != text: # real match (not fallback to full text) + if best_suffix is None: + best_suffix = s + elif prefer_long and len(s) > len(best_suffix): + best_suffix = s + elif not prefer_long and len(s) < len(best_suffix): + best_suffix = s + + if best_suffix is not None: + suffix = best_suffix + else: + # Fallback: use first reading + suffix = _extract_suffix(cd.src_map[0][0], cd.src_map[0][1]) + + steps.append(ConjStep( + conj_type=type_name, + suffix=suffix, + gloss=gloss, + neg=bool(cd.prop.neg), + fml=bool(cd.prop.fml), + )) + + +def _get_conj_suffix( + session: Session, + conj: Conjugation, + prop: ConjProp, +) -> str: + """ + Get the suffix text that represents this conjugation step. + + Uses ConjSourceReading to find the conjugated form, then extracts + the part that differs from the source. + """ + src_readings = session.execute( + select(ConjSourceReading).where(ConjSourceReading.conj_id == conj.id) + ).scalars().all() + + if not src_readings: + return "" + + # Pick the reading that gives the best suffix. + # Usually shortest non-empty to avoid variant kanji breaking extraction. + # Exception: Causative types prefer longest (standard form over dialectal). + prefer_long = prop.conj_type in (CONJ_CAUSATIVE, CONJ_CAUSATIVE_PASSIVE) + best_suffix = None + for sr in src_readings: + suffix = _extract_suffix(sr.text, sr.source_text) + if suffix and suffix != sr.text: # suffix != full text means we found a real match + if best_suffix is None: + best_suffix = suffix + elif prefer_long and len(suffix) > len(best_suffix): + best_suffix = suffix + elif not prefer_long and len(suffix) < len(best_suffix): + best_suffix = suffix + + if best_suffix is not None: + return best_suffix + + # Fallback: use first reading + return _extract_suffix(src_readings[0].text, src_readings[0].source_text) + + +def _split_neg_suffix(suffix: str) -> Tuple[str, str]: + """Split a negative conjugation suffix into (neg_part, conj_part). + + The negative marker ない conjugates like an i-adjective, so negative + suffixes contain the neg marker plus its own conjugation ending. + + Examples: + なかった → (ない, かった) # neg + past + まなかった → (まない, かった) # neg + past (godan) + くなかった → (くない, かった) # neg + past (i-adj) + なくて → (ない, くて) # neg + te + なければ → (ない, ければ) # neg + provisional + ないで → (ない, で) # neg + te (variant) + ない → (ない, "") # just negative + """ + # ない-conjugation endings (most specific first to avoid partial matches) + for ending in ("かったら", "かった", "ければ", "くて"): + if suffix.endswith(ending): + neg_part = suffix[:-len(ending)] + "い" + return neg_part, ending + + # ないで pattern (negative te-form variant) + if suffix.endswith("で") and len(suffix) >= 3: + potential_neg = suffix[:-1] + if potential_neg.endswith("ない"): + return potential_neg, "で" + + # No further conjugation (e.g., just ない) + return suffix, "" + + +def _extract_suffix(conj_text: str, src_text: str) -> str: + """ + Extract the changed suffix by comparing conjugated and source text. + + e.g., (食べられなかった, 食べられる) → なかった + (食べられる, 食べる) → られる (replace る with られる) + (行きました, 行く) → きました + """ + # Find common prefix + common_len = 0 + for i, (c1, c2) in enumerate(zip(conj_text, src_text)): + if c1 == c2: + common_len = i + 1 + else: + break + + suffix_part = conj_text[common_len:] + return suffix_part if suffix_part else conj_text diff --git a/himotoki/output/format.py b/himotoki/output/format.py new file mode 100644 index 0000000..0425aff --- /dev/null +++ b/himotoki/output/format.py @@ -0,0 +1,278 @@ +""" +Main segmentation output entry points and JSON formatting. +""" + +import json +from typing import Optional, List, Dict, Any, Union + +from sqlalchemy.orm import Session + +from himotoki.output.types import WordInfo, WordType, SPECIAL_CONJ_INFO +from himotoki.output.word_info import fill_segment_path +from himotoki.output.meanings import ( + word_info_reading_str, + get_senses_json, + get_entry_reading, + get_root_seq, + conj_info_json, +) +from himotoki.output.conjugation_display import _get_conjugation_display + +def word_info_gloss_json( + session: Session, + word_info: WordInfo, + root_only: bool = False, +) -> Dict[str, Any]: + """ + Generate JSON output for WordInfo. + + Args: + session: Database session + word_info: WordInfo to convert + root_only: If True, skip conjugation info + + Returns: + Dict ready for JSON serialization + """ + js = { + 'reading': word_info_reading_str(word_info), + 'text': word_info.text, + 'kana': word_info.kana, + } + + if word_info.score: + js['score'] = word_info.score + + if word_info.alternative: + # Multiple interpretations + js['alternative'] = [ + word_info_gloss_json(session, wi, root_only) + for wi in word_info.components + ] + return js + + if word_info.components: + # Compound word with component WordInfo objects + js['compound'] = [wi.text for wi in word_info.components] + js['components'] = [ + word_info_gloss_json(session, wi, root_only) + for wi in word_info.components + ] + return js + + # Handle compound words that don't have component WordInfo objects + # (e.g., from suffix-based compounds where we only have the text) + if word_info.is_compound: + seq = word_info.seq + if seq: + js['seq'] = seq + # Add gloss from the main word's seq + glosses = get_senses(session, seq) + if glosses: + js['gloss'] = glosses + + # Add compound texts if available (for ichiran compatibility) + if word_info.compound_texts: + js['compound'] = word_info.compound_texts + + # Build conjugation info directly from WordInfo fields + # (the conjugation data was already extracted in word_info_from_segment) + if word_info.conj_type: + conj_prop = { + 'type': word_info.conj_type, + } + if word_info.conj_neg: + conj_prop['neg'] = True + if word_info.conj_fml: + conj_prop['fml'] = True + + conj_entry = { + 'prop': [conj_prop], + 'readok': True, + } + + # Add source reading if available + if word_info.source_text: + conj_entry['reading'] = word_info.source_text + + js['conj'] = [conj_entry] + + return js + + if word_info.counter: + # Counter word + value, ordinal = word_info.counter + js['counter'] = {'value': value, 'ordinal': ordinal} + if word_info.seq: + js['seq'] = word_info.seq + gloss = get_senses_json(session, word_info.seq, pos_list=['ctr']) + if gloss: + js['gloss'] = gloss + return js + + # Regular word + seq = word_info.seq + if seq: + js['seq'] = seq + + if root_only or word_info.conjugations is None or word_info.conjugations == 'root': + gloss = get_senses_json(session, seq) + if gloss: + js['gloss'] = gloss + + # Get conjugation info + # Check for regular conjugations OR special conj info for standalone copulae + # Note: seq can be a list for compound words, so we need to check for hashability + has_conjugations = word_info.conjugations and word_info.conjugations != 'root' + has_special_conj = isinstance(seq, int) and seq in SPECIAL_CONJ_INFO + + if seq and (has_conjugations or has_special_conj): + conj = conj_info_json( + session, seq, + conjugations=word_info.conjugations if has_conjugations else None, + text=word_info.true_text, + ) + if conj: + js['conj'] = conj + + return js + +def dict_segment( + session: Session, + text: str, + limit: int = 5, +) -> List[tuple]: + """ + Segment text and return WordInfo results. + + This is the main entry point, equivalent to ichiran's dict-segment. + + Args: + session: Database session + text: Text to segment + limit: Maximum number of segmentation results + + Returns: + List of (word_info_list, score) tuples + """ + from himotoki.segment import segment_text + + results = segment_text(session, text, limit=limit) + + return [ + (fill_segment_path(session, text, path), score) + for path, score in results + ] + + +def simple_segment(session: Session, text: str, limit: int = 5) -> List[WordInfo]: + """ + Simple segmentation returning just the best path. + + Args: + session: Database session + text: Text to segment + limit: Maximum paths to consider + + Returns: + List of WordInfo for the best segmentation + """ + results = dict_segment(session, text, limit=limit) + if results: + return results[0][0] + return [] + + +def segment_to_json( + session: Session, + text: str, + limit: int = 5, +) -> List[List[Any]]: + """ + Segment text and return ichiran-compatible JSON. + + This matches the output format of ichiran-cli -f. + + Args: + session: Database session + text: Text to segment + limit: Maximum segmentation results + + Returns: + JSON-compatible nested list structure + """ + from himotoki.characters import romanize_word + + results = dict_segment(session, text, limit=limit) + + output = [] + for word_infos, score in results: + segments = [] + for wi in word_infos: + # [romanized, {word_info_json}, []] + # The third element is for split info (not yet implemented) + romanized = romanize_word(wi.kana if isinstance(wi.kana, str) else wi.kana[0] if wi.kana else wi.text) + segment_json = word_info_gloss_json(session, wi) + segments.append([romanized, segment_json, []]) + + output.append([segments, score]) + + return output + + +def segment_to_text( + session: Session, + text: str, + limit: int = 1, +) -> str: + """ + Segment text and return formatted text output. + + This matches the output format of ichiran-cli -i. + + Args: + session: Database session + text: Text to segment + limit: Maximum segmentation results + + Returns: + Formatted text output + """ + from himotoki.characters import romanize_word + + results = dict_segment(session, text, limit=limit) + + if not results: + return text + + word_infos, score = results[0] + + lines = [] + + # Romanized line + romanized_parts = [] + for wi in word_infos: + kana = wi.kana if isinstance(wi.kana, str) else wi.kana[0] if wi.kana else wi.text + romanized_parts.append(romanize_word(kana)) + lines.append(' '.join(romanized_parts)) + + # Word info lines + for wi in word_infos: + if wi.type == WordType.GAP: + continue + + lines.append('') + + romanized = romanize_word(wi.kana if isinstance(wi.kana, str) else wi.kana[0] if wi.kana else wi.text) + lines.append(f"* {romanized} {word_info_reading_str(wi)}") + + if wi.seq: + senses = get_senses_str(session, wi.seq) + lines.append(senses) + + # Conjugation info (breakdown tree) + conj_strs = _get_conjugation_display(session, wi) + for cs in conj_strs: + lines.append(cs) + + return '\n'.join(lines) diff --git a/himotoki/output/meanings.py b/himotoki/output/meanings.py new file mode 100644 index 0000000..ae5b1b2 --- /dev/null +++ b/himotoki/output/meanings.py @@ -0,0 +1,747 @@ +""" +Sense/gloss lookups, meanings cache, and reading helpers. +""" + +from dataclasses import dataclass +from typing import Optional, List, Dict, Any, Union + +from sqlalchemy import select, and_, func +from sqlalchemy.orm import Session + +from himotoki.db.models import ( + Entry, KanjiText, KanaText, Sense, Gloss, SenseProp, + Conjugation, ConjProp, +) +from himotoki.constants import get_conj_description +from himotoki.lookup import Segment, SegmentList +from himotoki.output.types import WordType, WordInfo, SPECIAL_CONJ_INFO + +def reading_str(kanji: Optional[str], kana: str) -> str: + """ + Format reading as 'kanji 【kana】' or just 'kana'. + + Args: + kanji: Kanji text (may be None) + kana: Kana reading + + Returns: + Formatted reading string + """ + if kanji: + return f"{kanji} 【{kana}】" + return kana + + +def get_entry_reading(session: Session, seq: int) -> str: + """Get formatted reading for an entry by seq.""" + kanji_text = session.execute( + select(KanjiText.text) + .where(and_(KanjiText.seq == seq, KanjiText.ord == 0)) + ).scalars().first() + + kana_text = session.execute( + select(KanaText.text) + .where(and_(KanaText.seq == seq, KanaText.ord == 0)) + ).scalars().first() + + return reading_str(kanji_text, kana_text or '') + + +def get_kana_for_entry(session: Session, seq: int) -> str: + """Get the primary kana reading for an entry by seq.""" + kana_text = session.execute( + select(KanaText.text) + .where(and_(KanaText.seq == seq, KanaText.ord == 0)) + ).scalars().first() + + return kana_text or '' + + +def get_matching_kana_for_kanji(session: Session, seq: int, kanji_text: str) -> str: + """ + Get kana reading that best matches the kanji text's suffix pattern. + + For conjugated forms, the DB may have multiple kana readings with different + suffixes (e.g., いわないで and いわなくて for 言わないで). We want the kana + whose suffix matches the kanji's suffix. + + Args: + session: Database session + seq: Entry sequence number + kanji_text: The matched kanji text (e.g., "言わないで") + + Returns: + The best matching kana reading + """ + # Get all kana readings for this entry + kana_results = session.execute( + select(KanaText.text, KanaText.ord) + .where(KanaText.seq == seq) + .order_by(KanaText.ord) + ).all() + + if not kana_results: + return '' + + if len(kana_results) == 1: + return kana_results[0][0] + + # For multiple readings, try to match suffix patterns + # Common confusing pairs: ないで/なくて, ないと/なくと + # Extract the kana suffix from kanji (last few hiragana chars) + kanji_suffix = '' + for char in reversed(kanji_text): + code = ord(char) + # Hiragana: 0x3040-0x309F + if 0x3040 <= code <= 0x309F: + kanji_suffix = char + kanji_suffix + else: + break + + if kanji_suffix: + # Find a kana reading that ends with the same suffix + for kana, ord_num in kana_results: + if kana.endswith(kanji_suffix): + return kana + + # Fall back to first reading if no suffix match + return kana_results[0][0] + + +# ============================================================================ +# Global Meanings Cache for Performance +# ============================================================================ + +# Global cache for meanings and POS data (persists across analyze() calls) +# Format: {seq: (meanings_list, pos_string)} +_MEANINGS_CACHE: Dict[int, tuple] = {} + +# Maximum cache size to prevent unbounded memory growth +_MEANINGS_CACHE_MAX_SIZE = 50000 + + +def get_cached_meanings(seq: int) -> Optional[tuple]: + """Get cached meanings for a seq if available.""" + return _MEANINGS_CACHE.get(seq) + + +def cache_meanings(seq: int, meanings: List[str], pos: Optional[str]) -> None: + """Cache meanings for a seq.""" + global _MEANINGS_CACHE + # Simple LRU-ish: if cache is too large, clear half of it + if len(_MEANINGS_CACHE) >= _MEANINGS_CACHE_MAX_SIZE: + # Keep the most recent half (dict preserves insertion order in Python 3.7+) + items = list(_MEANINGS_CACHE.items()) + _MEANINGS_CACHE = dict(items[len(items) // 2:]) + _MEANINGS_CACHE[seq] = (meanings, pos) + + +def clear_meanings_cache() -> None: + """Clear the global meanings cache (for testing).""" + global _MEANINGS_CACHE + _MEANINGS_CACHE = {} + + +# ============================================================================ +# Batch Preloading for Performance +# ============================================================================ + +class ReadingsCache: + """ + Cache for batch-loaded readings to avoid repeated DB queries. + + This significantly improves performance by loading all needed readings + in a single query instead of one query per word. + """ + + def __init__(self): + self.kanji_readings: Dict[int, str] = {} # seq -> primary kanji text + self.kana_readings: Dict[int, str] = {} # seq -> primary kana text + self._loaded = False + + def preload(self, session: Session, seqs: set) -> None: + """ + Batch load all kanji and kana readings for the given seqs. + + Args: + session: Database session + seqs: Set of seq numbers to load + """ + if not seqs: + return + + # Batch load kanji readings (ord=0 is primary) + kanji_results = session.execute( + select(KanjiText.seq, KanjiText.text) + .where(and_(KanjiText.seq.in_(seqs), KanjiText.ord == 0)) + ).all() + for seq, text in kanji_results: + self.kanji_readings[seq] = text + + # Batch load kana readings (ord=0 is primary) + kana_results = session.execute( + select(KanaText.seq, KanaText.text) + .where(and_(KanaText.seq.in_(seqs), KanaText.ord == 0)) + ).all() + for seq, text in kana_results: + self.kana_readings[seq] = text + + self._loaded = True + + def get_kanji(self, seq: int) -> Optional[str]: + """Get cached kanji reading for seq.""" + return self.kanji_readings.get(seq) + + def get_kana(self, seq: int) -> str: + """Get cached kana reading for seq.""" + return self.kana_readings.get(seq, '') + + def get_source_text(self, from_seq: int) -> Optional[str]: + """Get source text (dictionary form) for a conjugation source.""" + # Prefer kanji form, fall back to kana + kanji = self.kanji_readings.get(from_seq) + if kanji: + return kanji + return self.kana_readings.get(from_seq) + + +def collect_seqs_from_path(path: list) -> set: + """ + Collect all seq numbers needed from a path for batch preloading. + + This includes: + - Word seqs + - Conjugation from_seqs (for source_text lookup) + + Args: + path: List of SegmentLists or Segments + + Returns: + Set of seq numbers to preload + """ + from himotoki.lookup import CompoundWord + from himotoki.grammar.counters import CounterText + + seqs = set() + + for item in path: + if isinstance(item, SegmentList): + # Get seqs from all segments in the list + for segment in item.segments: + _collect_segment_seqs(segment, seqs) + elif isinstance(item, Segment): + _collect_segment_seqs(item, seqs) + + return seqs + + +def _collect_segment_seqs(segment: Segment, seqs: set) -> None: + """Helper to collect seqs from a single segment.""" + from himotoki.lookup import CompoundWord + from himotoki.grammar.counters import CounterText + + word = segment.word + + # Skip CounterText - they don't have entries + if isinstance(word, CounterText): + if word.seq: + seqs.add(word.seq) + return + + # Add word seq + if hasattr(word, 'seq') and word.seq: + seqs.add(word.seq) + + # For CompoundWords, add all component seqs + if isinstance(word, CompoundWord): + if word.primary and hasattr(word.primary, 'seq'): + seqs.add(word.primary.seq) + for w in word.words: + if hasattr(w, 'seq') and w.seq: + seqs.add(w.seq) + + # Add from_seqs from conjugation data + conj_data = segment.info.get('conj', []) if segment.info else [] + for cd in conj_data: + if cd.from_seq: + seqs.add(cd.from_seq) + + +def word_info_reading_str(word_info: WordInfo) -> str: + """Get formatted reading string for WordInfo. + + For compounds where する is absorbed, merges the kana + (e.g., 'べんきょう しています' → 'べんきょうしています'). + """ + if word_info.type == WordType.KANJI or word_info.counter: + kana = word_info.kana + if isinstance(kana, list): + kana = '/'.join(kana) + # Merge kana when する is absorbed into a noun compound + if word_info.is_compound and word_info.components and isinstance(kana, str): + kana = _merge_suru_kana(word_info, kana) + return reading_str(word_info.text, kana) + return reading_str(None, word_info.text) + + +def _merge_suru_kana(word_info: WordInfo, kana: str) -> str: + """Merge kana spaces when する is absorbed into a noun compound. + + When the first suffix component is する (or a conjugated form of する), + remove the space between the noun kana and the verb kana. + + 'べんきょう しています' → 'べんきょうしています' + 'べんきょう した' → 'べんきょうした' + """ + if not word_info.components or len(word_info.components) < 2: + return kana + + primary = word_info.components[0] + suffix = word_info.components[1] + + # Only absorb plain する (not させる, される, いたす, etc.) + # Check if primary has no conjugation (it's a noun) + if primary.conjugations and primary.conjugations != 'root': + return kana + + suffix_kana = suffix.kana if isinstance(suffix.kana, str) else (suffix.kana[0] if suffix.kana else suffix.text) + # する or conjugated forms start with し or す + if suffix_kana and (suffix_kana.startswith("し") or suffix_kana.startswith("す")): + # Verify it's actually する-based by checking the kana pattern + # Valid: する, して, した, します, しない, しました, etc. + # Invalid: しまう (different verb), すぐ (unrelated) + primary_kana = primary.kana if isinstance(primary.kana, str) else (primary.kana[0] if primary.kana else primary.text) + expected_prefix = primary_kana + " " + suffix_kana[0] + if primary_kana and kana.startswith(expected_prefix): + return kana.replace(primary_kana + " ", primary_kana, 1) + + return kana + + +def has_conjugable_pos(session: Session, seq: int) -> bool: + """ + Check if a word has a part-of-speech that can be conjugated. + + This is used to determine if a dictionary form should have conj_type="Non-past". + Only returns True for verbs and i-adjectives which have true conjugation forms. + + Args: + session: Database session + seq: Entry sequence number + + Returns: + True if the word has a conjugable POS (verb or i-adjective) + """ + if seq is None: + return False + + # Parts of speech that should show "Non-past" for dictionary forms + # This is more restrictive than POS_WITH_CONJ_RULES because we only want + # verbs and i-adjectives that have true conjugated forms. + # Excludes: + # - adj-na, adj-no (which use copula for conjugation) + # - vs (verbal nouns that take する - the noun part doesn't conjugate) + NONPAST_POS = frozenset([ + 'v1', 'v1-s', 'v1s', 'v5aru', 'v5b', 'v5g', 'v5k', 'v5k-s', 'v5m', 'v5n', + 'v5r', 'v5r-i', 'v5s', 'v5t', 'v5u', 'v5u-s', 'v5uru', 'vk', + 'vs-i', 'vs-s', 'vz', 'adj-i', 'adj-ix', + 'cop', 'cop-da', 'aux-v', # Copula and auxiliary verbs + ]) + + # Query for POS tags + pos_results = session.execute( + select(SenseProp.text) + .join(Sense, SenseProp.sense_id == Sense.id) + .where(and_( + Sense.seq == seq, + SenseProp.tag == 'pos', + )) + ).scalars().all() + + # Check if any POS is in the conjugable set + for pos in pos_results: + if pos in NONPAST_POS: + return True + + return False +# ============================================================================ +# Sense/Gloss Functions +# ============================================================================ + +def get_senses_raw(session: Session, seq: Union[int, List[int]]) -> List[Dict[str, Any]]: + """ + Get raw sense data for an entry. + + Args: + session: Database session + seq: Entry sequence number or list of sequence numbers (for compound words) + + Returns: + List of sense dicts with ord, gloss, and props + """ + tags = ['pos', 's_inf', 'stagk', 'stagr', 'field'] + + # Handle list of seqs (compound words) - use first seq for senses + if isinstance(seq, list): + if not seq: + return [] + seq = seq[0] + + # Get glosses grouped by sense + glosses_query = ( + select(Sense.ord, func.group_concat(Gloss.text, '; ')) + .join(Gloss, Gloss.sense_id == Sense.id, isouter=True) + .where(Sense.seq == seq) + .group_by(Sense.id) + .order_by(Sense.ord) + ) + glosses = session.execute(glosses_query).all() + + # Get properties + props_query = ( + select(Sense.ord, SenseProp.tag, SenseProp.text) + .join(SenseProp, SenseProp.sense_id == Sense.id) + .where(and_(Sense.seq == seq, SenseProp.tag.in_(tags))) + .order_by(Sense.ord, SenseProp.tag, SenseProp.ord) + ) + props = session.execute(props_query).all() + + # Build sense list + sense_list = [ + {'ord': ord_val, 'gloss': gloss or '', 'props': {}} + for ord_val, gloss in glosses + ] + + # Organize props by sense and tag + for sord, tag, text in props: + for sense in sense_list: + if sense['ord'] == sord: + if tag not in sense['props']: + sense['props'][tag] = [] + sense['props'][tag].append(text) + break + + return sense_list + + +def get_senses(session: Session, seq: Union[int, List[int]]) -> List[Dict[str, Any]]: + """ + Get senses formatted for output. + + Args: + session: Database session + seq: Entry sequence number or list (for compound words) + + Returns list of dicts with pos_str, gloss, and props. + """ + result = [] + for sense in get_senses_raw(session, seq): + props = sense['props'] + pos = props.get('pos', []) + pos_str = f"[{','.join(pos)}]" if pos else '[]' + result.append({ + 'pos': pos_str, + 'gloss': sense['gloss'], + 'props': props, + }) + return result + + +def get_root_seq(session: Session, seq: int) -> Optional[int]: + """Get the root (dictionary form) seq for a conjugated entry. + + Walks the conjugation table to find the from_seq (root entry). + Returns None if no conjugation data found. + """ + conjs = session.execute( + select(Conjugation).where(Conjugation.seq == seq) + ).scalars().all() + if conjs: + return conjs[0].from_seq + return None + + +def get_senses_str(session: Session, seq: Union[int, List[int]]) -> str: + """Get senses as formatted string. + + If the entry has no senses (e.g., a conjugation-only entry), + returns an empty string. + """ + lines = [] + rpos = '[]' + + for i, sense in enumerate(get_senses(session, seq), 1): + pos = sense['pos'] + if pos != '[]': + rpos = pos + + gloss = sense['gloss'] + props = sense['props'] + + info = props.get('s_inf', []) + rinf = '; '.join(info) if info else None + + fields = props.get('field', []) + rfield = ','.join(fields) if fields else None + + parts = [f"{i}. {rpos}"] + if rfield: + parts.append(f"{{{rfield}}}") + if rinf: + parts.append(f"《{rinf}》") + parts.append(gloss) + + lines.append(' '.join(parts)) + + return '\n'.join(lines) + + +def get_senses_json( + session: Session, + seq: int, + pos_list: Optional[List[str]] = None, + reading: Optional[Any] = None, +) -> List[Dict[str, Any]]: + """ + Get senses as JSON-compatible dicts. + + Args: + session: Database session + seq: Entry sequence number + pos_list: Filter to these POS tags + reading: Filter to senses matching this reading + + Returns: + List of sense dicts for JSON output + """ + result = [] + rpos = '[]' + + for sense in get_senses(session, seq): + pos = sense['pos'] + if pos != '[]': + rpos = pos + + # POS filtering + if pos_list: + lpos = pos[1:-1].split(',') if pos != '[]' else [] + if not any(p in pos_list for p in lpos): + continue + + gloss = sense['gloss'] + props = sense['props'] + + js = {'pos': rpos, 'gloss': gloss} + + # Add field info + fields = props.get('field', []) + if fields: + js['field'] = f"{{{','.join(fields)}}}" + + # Add sense info + info = props.get('s_inf', []) + if info: + js['info'] = '; '.join(info) + + result.append(js) + + return result + + +# ============================================================================ +# Conjugation Info Functions +# ============================================================================ + +# get_conj_description is now imported from himotoki.constants + + +def conj_prop_json(prop: ConjProp) -> Dict[str, Any]: + """Convert conjugation property to JSON dict.""" + js = { + 'pos': prop.pos, + 'type': get_conj_description(prop.conj_type), + } + if prop.neg: + js['neg'] = True + if prop.fml: + js['fml'] = True + return js + + +def conj_info_json( + session: Session, + seq: int, + conjugations: Optional[List[int]] = None, + text: Optional[str] = None, +) -> List[Dict[str, Any]]: + """ + Get conjugation info as JSON. + + Args: + session: Database session + seq: Entry sequence number + conjugations: Specific conjugation IDs to include + text: Filter by text + + Returns: + List of conjugation info dicts + """ + result = [] + + # Get conjugations + query = select(Conjugation).where(Conjugation.seq == seq) + if conjugations and conjugations != 'root': + query = query.where(Conjugation.id.in_(conjugations)) + + conjs = session.execute(query).scalars().all() + + for conj in conjs: + # Get properties + props = session.execute( + select(ConjProp).where(ConjProp.conj_id == conj.id) + ).scalars().all() + + if not props: + continue + + js = { + 'prop': [conj_prop_json(p) for p in props], + 'reading': get_entry_reading(session, conj.from_seq), + 'gloss': get_senses_json(session, conj.from_seq), + 'readok': True, + } + + result.append(js) + + # Check for special conj info for standalone entries that represent conjugated forms + if not result and seq in SPECIAL_CONJ_INFO: + from_seq, conj_type, pos, neg, fml = SPECIAL_CONJ_INFO[seq] + js = { + 'prop': [{ + 'pos': pos, + 'type': get_conj_description(conj_type), + 'neg': neg, + 'fml': fml, + }], + 'reading': get_entry_reading(session, from_seq), + 'gloss': get_senses_json(session, from_seq), + 'readok': True, + } + result.append(js) + + return result + +def _split_copula_compound_for_output(wi: WordInfo) -> List[WordInfo]: + """Split na-adjective/noun + copula compounds for display output. + + Keeps internal parsing/scoring behavior unchanged while presenting + user-facing output as separate tokens (e.g., 綺麗 + です + ね). + """ + if not wi.is_compound or not wi.components or len(wi.components) != 2: + return [wi] + + primary, suffix = wi.components + suffix_kana = suffix.kana if isinstance(suffix.kana, str) else (suffix.kana[0] if suffix.kana else suffix.text) + if suffix_kana not in ('です', 'でした'): + return [wi] + + # Only split when primary itself is not conjugated (noun/na-adj style usage) + if primary.conjugations and primary.conjugations != 'root': + return [wi] + + return [primary, suffix] + + +def populate_meanings(session: Session, word_infos: List[WordInfo]) -> None: + """ + Populate meanings and pos fields for a list of WordInfo objects. + + Uses a global cache to avoid repeated DB queries for the same words. + Only queries the database for seqs not already in cache. + + Args: + session: Database session + word_infos: List of WordInfo objects to populate + """ + # Collect all seqs that need meanings, checking cache first + seq_to_words: Dict[int, List[WordInfo]] = {} + uncached_seqs: List[int] = [] + + for wi in word_infos: + if wi.seq and wi.type != WordType.GAP: + # Handle list seqs (compound words) - use first seq + seq = wi.seq[0] if isinstance(wi.seq, list) else wi.seq + if seq not in seq_to_words: + seq_to_words[seq] = [] + # Check if we already have this in cache + cached = get_cached_meanings(seq) + if cached is None: + uncached_seqs.append(seq) + seq_to_words[seq].append(wi) + + if not seq_to_words: + return + + # Build lookup dicts - start with empty, will populate from cache or DB + meanings_by_seq: Dict[int, List[str]] = {} + pos_by_seq: Dict[int, Optional[str]] = {} + + # Populate from cache for already-cached seqs + for seq in seq_to_words.keys(): + cached = get_cached_meanings(seq) + if cached is not None: + meanings_by_seq[seq] = cached[0] + pos_by_seq[seq] = cached[1] + + # Only query DB for uncached seqs + if uncached_seqs: + # Query all senses and glosses in one go + senses_query = ( + select(Sense.seq, Sense.ord, func.group_concat(Gloss.text, '; ')) + .join(Gloss, Gloss.sense_id == Sense.id, isouter=True) + .where(Sense.seq.in_(uncached_seqs)) + .group_by(Sense.id) + .order_by(Sense.seq, Sense.ord) + ) + senses_results = session.execute(senses_query).all() + + # Query POS for all uncached seqs + pos_query = ( + select(Sense.seq, SenseProp.text) + .join(SenseProp, SenseProp.sense_id == Sense.id) + .where(and_(Sense.seq.in_(uncached_seqs), SenseProp.tag == 'pos', Sense.ord == 0)) + .order_by(Sense.seq, SenseProp.ord) + ) + pos_results = session.execute(pos_query).all() + + # Build meanings by seq from DB results + for seq, ord_val, gloss in senses_results: + if seq not in meanings_by_seq: + meanings_by_seq[seq] = [] + if gloss: + meanings_by_seq[seq].append(gloss) + + # Build POS by seq from DB results + pos_by_seq_temp: Dict[int, List[str]] = {} + for seq, pos_text in pos_results: + if seq not in pos_by_seq_temp: + pos_by_seq_temp[seq] = [] + pos_by_seq_temp[seq].append(pos_text) + + for seq, tags in pos_by_seq_temp.items(): + pos_by_seq[seq] = f"[{','.join(tags)}]" + + # Cache the newly loaded data + for seq in uncached_seqs: + meanings = meanings_by_seq.get(seq, []) + pos = pos_by_seq.get(seq) + cache_meanings(seq, meanings, pos) + + # Populate WordInfo objects + for seq, word_list in seq_to_words.items(): + meanings = meanings_by_seq.get(seq, []) + pos = pos_by_seq.get(seq) + for wi in word_list: + wi.meanings = meanings + wi.pos = pos + diff --git a/himotoki/output/types.py b/himotoki/output/types.py new file mode 100644 index 0000000..5b59db1 --- /dev/null +++ b/himotoki/output/types.py @@ -0,0 +1,127 @@ +""" +Output types and constants. +""" + +from dataclasses import dataclass, field +from typing import Optional, List, Dict, Any, Union +from enum import Enum + +# ============================================================================ +# Enums and Constants +# ============================================================================ + +class WordType(Enum): + """Word type classification.""" + KANJI = 'kanji' + KANA = 'kana' + GAP = 'gap' + + +# Special conjugation info for entries that are standalone but represent +# conjugated forms of other entries (like です being formal non-past of だ) +# Format: seq -> (from_seq, conj_type, pos, neg, fml) +SPECIAL_CONJ_INFO: Dict[int, tuple] = { + 1628500: (2089020, 1, 'cop', False, True), # です = non-past formal of だ +} + +# Particle seqs that should NOT display conjugation info even if they match +# conjugated forms. These are standalone particles that happen to look like +# conjugated forms but should be treated as particles in most contexts. +# Example: で (seq 2028980) can be copula だ conjunctive form, but when it's +# matched as a standalone word, it's almost always the case particle. +SUPPRESS_CONJ_FOR_PARTICLES: set = { + 2028980, # で - case particle (location/means), not copula て-form +} + +# Words that have their own standalone dictionary meaning and should NOT +# display as conjugated forms of other verbs, even though the DB links them. +# These are typically nominalized verb forms that became independent nouns. +SUPPRESS_CONJ_FOR_NOUNS: set = { + 1382980, # 積もり (つもり) - intention/plan noun, not continuative of 積もる +} + +# Verbs that have their own dictionary entry but might be mistakenly identified +# as potential or other conjugated forms of different verbs. These should show +# their own glosses, not conjugation info from another verb. +SUPPRESS_CONJ_FOR_VERBS: set = { + 1345930, # 傷つける - standalone transitive v1, not potential of 傷つく +} + + +# ============================================================================ +# Data Classes +# ============================================================================ + +@dataclass +class WordInfo: + """ + Word information for output. + Mirrors ichiran's word-info class. + """ + type: WordType # :kanji, :kana, or :gap + text: str # Surface text + kana: Union[str, List[str]] # Reading(s) + + # Optional fields + true_text: Optional[str] = None # Original text (for proxy text) + seq: Optional[Union[int, List[int]]] = None # JMdict sequence number(s) + conjugations: Optional[Union[List[int], str]] = None # Conjugation IDs or 'root' + score: int = 0 + components: List['WordInfo'] = field(default_factory=list) # For compound words (WordInfo objects) + compound_texts: List[str] = field(default_factory=list) # Component texts for suffix compounds + alternative: bool = False # True if multiple readings available + primary: bool = True # Is this the primary reading + start: Optional[int] = None + end: Optional[int] = None + counter: Optional[List[Any]] = None # [value, ordinal] for counter words + skipped: int = 0 # Number of skipped alternatives + + # Conjugation info fields + is_compound: bool = False # True if this is a compound word + conj_type: Optional[str] = None # Human-readable conjugation type + conj_neg: bool = False # True if negative form + conj_fml: bool = False # True if formal/polite form + source_text: Optional[str] = None # Dictionary form for conjugated words + + # NEW: Meanings and POS - populated during analysis + meanings: List[str] = field(default_factory=list) # List of gloss strings + pos: Optional[str] = None # Part of speech (e.g., "[n,vs,vt]") + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary (for JSON serialization).""" + return { + 'type': self.type.value.upper(), + 'text': self.text, + 'truetext': self.true_text, + 'kana': self.kana, + 'seq': self.seq, + 'conjugations': 'ROOT' if self.conjugations == 'root' else self.conjugations, + 'score': self.score, + 'components': [c.to_dict() for c in self.components] if self.components else [], + 'compound_texts': self.compound_texts, + 'alternative': self.alternative, + 'primary': self.primary, + 'start': self.start, + 'end': self.end, + 'counter': self.counter, + 'skipped': self.skipped, + 'is_compound': self.is_compound, + 'conj_type': self.conj_type, + 'conj_neg': self.conj_neg, + 'conj_fml': self.conj_fml, + 'source_text': self.source_text, + 'meanings': self.meanings, + 'pos': self.pos, + } + + +@dataclass +class ConjStep: + """One step in a conjugation breakdown chain.""" + conj_type: str # e.g., "Passive", "Past (~ta)" + suffix: str # e.g., "られる", "た" + gloss: str # e.g., "is done (to)", "did/was" + neg: bool = False # True if this step is negative + fml: bool = False # True if this step is formal/polite + + diff --git a/himotoki/output/word_info.py b/himotoki/output/word_info.py new file mode 100644 index 0000000..090d38e --- /dev/null +++ b/himotoki/output/word_info.py @@ -0,0 +1,524 @@ +""" +WordInfo creation from segments and path filling helpers. +""" + +from typing import Optional, List, Dict, Any, Union + +from sqlalchemy import select, and_ +from sqlalchemy.orm import Session + +from himotoki.db.models import KanjiText, KanaText +from himotoki.raw_types import RawKanaReading, RawKanjiReading +from himotoki.lookup import ( + Segment, SegmentList, WordMatch, + get_conj_type_name, get_conj_neg, get_conj_fml, get_source_text, + POS_WITH_CONJ_RULES, +) +from himotoki.constants import CONJ_TYPE_NAMES +from himotoki.output.types import WordType, WordInfo, SUPPRESS_CONJ_FOR_PARTICLES, SUPPRESS_CONJ_FOR_NOUNS, SUPPRESS_CONJ_FOR_VERBS +from himotoki.output.meanings import ( + ReadingsCache, + reading_str, + get_matching_kana_for_kanji, + has_conjugable_pos, + collect_seqs_from_path, + _collect_segment_seqs, + populate_meanings, + _split_copula_compound_for_output, +) + +def word_info_from_word_match( + session: Session, + word_match, + cache: Optional[ReadingsCache] = None, +) -> WordInfo: + """ + Create a simple WordInfo from a WordMatch object. + Used for creating component WordInfo objects in compound words. + + Args: + session: Database session + word_match: WordMatch object from suffix/split compounds + cache: Optional preloaded readings cache for performance + + Returns: + WordInfo object with basic info (text, kana, seq, conjugations) + """ + reading = word_match.reading + + # Determine word type and kana + if word_match.word_type == 'kanji': + word_type = WordType.KANJI + # Get kana from best_kana or matching function + kana = getattr(reading, 'best_kana', None) + if not kana: + kana = get_matching_kana_for_kanji(session, word_match.seq, reading.text) + if not kana: + # Last resort - use empty string + kana = '' + else: + word_type = WordType.KANA + kana = reading.text + + # Get conjugation info if available + conj_type_name = None + source_text = None + + # If there are conjugations, try to get the first one's info + if word_match.conjugations and word_match.conjugations != 'root': + conj_ids = word_match.conjugations if isinstance(word_match.conjugations, list) else [word_match.conjugations] + if conj_ids: + conj_id = conj_ids[0] + # Look up conjugation info - Conjugation table, then get props + from himotoki.db.models import Conjugation + conj = session.query(Conjugation).filter(Conjugation.id == conj_id).first() + if conj: + # Get conj_type from props relationship (not directly on Conjugation) + if conj.props: + prop = conj.props[0] + conj_type_name = CONJ_TYPE_NAMES.get(prop.conj_type) + # Get source text from from_seq + if conj.from_seq: + if cache: + source_text = cache.get_source_text(conj.from_seq) + else: + kanji_text = session.execute( + select(KanjiText.text) + .where(and_(KanjiText.seq == conj.from_seq, KanjiText.ord == 0)) + ).scalars().first() + source_text = kanji_text + else: + # No conjugation data - this is a dictionary form + # Set conj_type to "Non-past" for verbs and adjectives + if has_conjugable_pos(session, word_match.seq): + conj_type_name = "Non-past" + + return WordInfo( + type=word_type, + text=word_match.text, + kana=kana, + seq=word_match.seq, + conjugations=word_match.conjugations if word_match.conjugations != 'root' else None, + conj_type=conj_type_name, + source_text=source_text, + ) + + +def word_info_from_segment( + session: Session, + segment: Segment, + cache: Optional[ReadingsCache] = None, +) -> WordInfo: + """ + Create WordInfo from a segment. + + Args: + session: Database session + segment: Segment with word match + cache: Optional preloaded readings cache for performance + + Returns: + WordInfo object + """ + from himotoki.lookup import CompoundWord + from himotoki.grammar.counters import CounterText + + word = segment.word + + # Handle CounterText specially + if isinstance(word, CounterText): + word_type = WordType.KANJI # Counters are typically kanji + return WordInfo( + type=word_type, + text=segment.get_text(), + kana=word.kana, + true_text=word.text, + seq=word.seq, + conjugations=[], + score=int(segment.score), + start=segment.start, + end=segment.end, + ) + + # Handle CompoundWord specially + if isinstance(word, CompoundWord): + word_type = WordType.KANA if word.word_type == 'kana' else WordType.KANJI + + # Get conjugation data from segment.info (computed by calc_score) + # This is the authoritative source for conjugation info + conj_data = segment.info.get('conj', []) if segment.info else [] + + # Extract conjugation IDs from conj_data + # Use conj_id (foreign key to Conjugation.id), not prop.id + conjugations = None + if conj_data: + conj_ids = [cd.prop.conj_id if cd.prop else None for cd in conj_data] + conj_ids = [cid for cid in conj_ids if cid is not None] + conjugations = conj_ids if conj_ids else None + + # Extract conjugation info directly from conj_data + conj_type_name = None + conj_neg = False + conj_fml = False + source_text = None + + if conj_data: + # Get the first conjugation data entry (primary conjugation) + cd = conj_data[0] + if cd.prop: + # Get conj_type name from the mapping + conj_type_name = CONJ_TYPE_NAMES.get(cd.prop.conj_type) + conj_neg = bool(cd.prop.neg) + conj_fml = bool(cd.prop.fml) + + # Get source_text - prefer kanji form from from_seq, fall back to src_map + # This matches ichiran's behavior of showing the dictionary form + if cd.from_seq: + # Use cache if available, otherwise query DB + if cache: + source_text = cache.get_source_text(cd.from_seq) + else: + # Try to get kanji form first + kanji_text = session.execute( + select(KanjiText.text) + .where(and_(KanjiText.seq == cd.from_seq, KanjiText.ord == 0)) + ).scalars().first() + if kanji_text: + source_text = kanji_text + else: + # Fall back to kana form + kana_text = session.execute( + select(KanaText.text) + .where(and_(KanaText.seq == cd.from_seq, KanaText.ord == 0)) + ).scalars().first() + if kana_text: + source_text = kana_text + else: + # Fallback: get conjugation info from the CompoundWord itself + # This handles suffix-created compounds where segment.info doesn't have conj data + conj_info = word.get_conjugation_info(session) + if conj_info.get('conj_type') is not None: + conj_type_name = CONJ_TYPE_NAMES.get(conj_info['conj_type']) + conj_neg = bool(conj_info.get('neg', False)) + conj_fml = bool(conj_info.get('fml', False)) + source_text = conj_info.get('source_text') + + # Also get conjugation IDs from the last word + if word.conjugations: + conjugations = word.conjugations + + # Get component texts from CompoundWord.components property + # This returns the text of each word in the compound + compound_texts = word.components if word.components else [] + + # Create component WordInfo objects from the underlying WordMatch objects + # This provides detailed info (including POS) for each part of the compound + component_word_infos = [] + if hasattr(word, 'words') and word.words: + for wm in word.words: + try: + comp_wi = word_info_from_word_match(session, wm, cache) + component_word_infos.append(comp_wi) + except Exception: + # If we fail to create a component WordInfo, skip it + pass + + return WordInfo( + type=word_type, + text=segment.get_text(), + kana=word.kana, # Use compound's full kana + true_text=word.text, + seq=word.seq, # This is an int for compound words (primary's seq) + conjugations=conjugations, + score=int(segment.score), + components=component_word_infos, # NEW: Component WordInfo objects + start=segment.start, + end=segment.end, + is_compound=True, + compound_texts=compound_texts, + conj_type=conj_type_name, + conj_neg=conj_neg, + conj_fml=conj_fml, + source_text=source_text, + ) + + reading = word.reading + + # Determine kana reading + # Handle both ORM objects (KanjiText/KanaText) and raw namedtuples (RawKanjiReading/RawKanaReading) + if isinstance(reading, (KanjiText, RawKanjiReading)): + word_type = WordType.KANJI + # Get best kana for kanji text - try best_kana attr, then suffix matching + kana = reading.best_kana + if not kana: + # Use suffix matching to get correct kana (e.g., ないで vs なくて) + kanji_text = reading.text + kana = get_matching_kana_for_kanji(session, word.seq, kanji_text) + else: + word_type = WordType.KANA + kana = reading.text + + # Get conjugation data from segment.info (computed by calc_score) + # This is the authoritative source for conjugation info + conj_data = segment.info.get('conj', []) if segment.info else [] + + # Suppress conjugation info for standalone particles + # These words have their own dictionary meaning as particles, and + # showing conjugation info (e.g., で as copula て-form) is misleading + if word.seq in SUPPRESS_CONJ_FOR_PARTICLES: + conj_data = [] + + # Suppress conjugation info for nouns that have their own dictionary meaning + # These are nominalized verb forms that became independent nouns + if word.seq in SUPPRESS_CONJ_FOR_NOUNS: + conj_data = [] + + # Suppress conjugation info for verbs with their own dictionary entries + # These might look like conjugated forms but are standalone verbs + if word.seq in SUPPRESS_CONJ_FOR_VERBS: + conj_data = [] + + # Extract conjugation IDs for the conjugations field + conjugations = word.conjugations + if conjugations is None and conj_data: + # Use conj_id (foreign key to Conjugation.id), not prop.id + # This matches ichiran's (conj-id (conj-data-prop cdata)) + conj_ids = [cd.prop.conj_id if cd.prop else None for cd in conj_data] + conj_ids = [cid for cid in conj_ids if cid is not None] + conjugations = conj_ids if conj_ids else None + + # Extract conjugation info directly from conj_data + # This avoids re-querying the database and ensures we use the same data + conj_type_name = None + conj_neg = False + conj_fml = False + source_text = None + + if conj_data: + # Get the first conjugation data entry (primary conjugation) + cd = conj_data[0] + if cd.prop: + # Get conj_type name from the mapping + conj_type_name = CONJ_TYPE_NAMES.get(cd.prop.conj_type) + conj_neg = bool(cd.prop.neg) + conj_fml = bool(cd.prop.fml) + + # Get source_text - prefer kanji form from from_seq, fall back to src_map + # This matches ichiran's behavior of showing the dictionary form + if cd.from_seq: + # Use cache if available, otherwise query DB + if cache: + source_text = cache.get_source_text(cd.from_seq) + else: + # Try to get kanji form first + kanji_text = session.execute( + select(KanjiText.text) + .where(and_(KanjiText.seq == cd.from_seq, KanjiText.ord == 0)) + ).scalars().first() + if kanji_text: + source_text = kanji_text + else: + # Fall back to kana form + kana_text = session.execute( + select(KanaText.text) + .where(and_(KanaText.seq == cd.from_seq, KanaText.ord == 0)) + ).scalars().first() + if kana_text: + source_text = kana_text + else: + # No conjugation data - this is a dictionary form + # Set conj_type to "Non-past" for verbs and adjectives + if has_conjugable_pos(session, word.seq): + conj_type_name = "Non-past" + + return WordInfo( + type=word_type, + text=segment.get_text(), + kana=kana, + true_text=word.text, + seq=word.seq, + conjugations=conjugations, + score=int(segment.score), + start=segment.start, + end=segment.end, + is_compound=False, + conj_type=conj_type_name, + conj_neg=conj_neg, + conj_fml=conj_fml, + source_text=source_text, + ) + + +def word_info_from_segment_list( + session: Session, + segment_list: SegmentList, + cache: Optional[ReadingsCache] = None, +) -> WordInfo: + """ + Create WordInfo from a segment list (multiple interpretations). + + Args: + session: Database session + segment_list: SegmentList with multiple segments + cache: Optional preloaded readings cache for performance + + Returns: + WordInfo object (possibly with alternatives) + """ + segments = segment_list.segments + + if not segments: + return WordInfo( + type=WordType.GAP, + text='', + kana='', + start=segment_list.start, + end=segment_list.end, + ) + + # Create WordInfo for each segment (pass cache) + wi_list = [word_info_from_segment(session, seg, cache) for seg in segments] + wi1 = wi_list[0] + max_score = wi1.score + + # Filter out low-scoring alternatives + cutoff = max_score * 0.67 # SEGMENT_SCORE_CUTOFF + wi_list = [wi for wi in wi_list if wi.score >= cutoff] + + matches = segment_list.matches + + if len(wi_list) == 1: + wi1.skipped = matches - 1 + return wi1 + + # Multiple alternatives + kana_list = [] + seq_list = [] + for wi in wi_list: + if isinstance(wi.kana, list): + kana_list.extend(wi.kana) + else: + kana_list.append(wi.kana) + if wi.seq: + if isinstance(wi.seq, list): + seq_list.extend(wi.seq) + else: + seq_list.append(wi.seq) + + # Remove duplicates while preserving order + seen_kana = set() + unique_kana = [] + for k in kana_list: + if k not in seen_kana: + unique_kana.append(k) + seen_kana.add(k) + + return WordInfo( + type=wi1.type, + text=wi1.text, + kana=unique_kana if len(unique_kana) > 1 else (unique_kana[0] if unique_kana else ''), + seq=seq_list if len(seq_list) > 1 else (seq_list[0] if seq_list else None), + components=wi_list, + alternative=True, + score=wi1.score, + start=segment_list.start, + end=segment_list.end, + skipped=matches - len(wi_list), + # Propagate conjugation info from primary alternative + conjugations=wi1.conjugations, + conj_type=wi1.conj_type, + conj_neg=wi1.conj_neg, + conj_fml=wi1.conj_fml, + source_text=wi1.source_text, + ) + + +def word_info_from_text(text: str) -> WordInfo: + """Create gap WordInfo for unmatched text.""" + return WordInfo( + type=WordType.GAP, + text=text, + kana=text, + ) +def fill_segment_path( + session: Session, + text: str, + path: List[SegmentList], + include_meanings: bool = True, +) -> List[WordInfo]: + """ + Fill gaps in segment path and convert to WordInfo list. + + Args: + session: Database session + text: Original text + path: List of SegmentLists or Segments from find_best_path + include_meanings: If True, populate meanings field (default True) + + Returns: + List of WordInfo objects covering the entire text + """ + # Batch preload all readings for performance + # This reduces N queries to 2 queries (one for kanji, one for kana) + seqs = collect_seqs_from_path(path) + cache = ReadingsCache() + cache.preload(session, seqs) + + result = [] + idx = 0 + + for item in path: + if isinstance(item, SegmentList): + segment_list = item + # Add gap before this segment if needed + if segment_list.start > idx: + gap_text = text[idx:segment_list.start] + result.append(WordInfo( + type=WordType.GAP, + text=gap_text, + kana=gap_text, + start=idx, + end=segment_list.start, + )) + + # Add the segment (pass cache for optimized lookups) + wi = word_info_from_segment_list(session, segment_list, cache) + expanded = _split_copula_compound_for_output(wi) + result.extend(expanded) + idx = segment_list.end + elif isinstance(item, Segment): + segment = item + # Add gap before this segment if needed + if segment.start > idx: + gap_text = text[idx:segment.start] + result.append(WordInfo( + type=WordType.GAP, + text=gap_text, + kana=gap_text, + start=idx, + end=segment.start, + )) + + # Add the segment (pass cache for optimized lookups) + wi = word_info_from_segment(session, segment, cache) + expanded = _split_copula_compound_for_output(wi) + result.extend(expanded) + idx = segment.end + + # Add trailing gap if needed + if idx < len(text): + gap_text = text[idx:] + result.append(WordInfo( + type=WordType.GAP, + text=gap_text, + kana=gap_text, + start=idx, + end=len(text), + )) + + # Populate meanings if requested + if include_meanings: + populate_meanings(session, result) + + return result + diff --git a/himotoki/scoring/__init__.py b/himotoki/scoring/__init__.py new file mode 100644 index 0000000..2101c94 --- /dev/null +++ b/himotoki/scoring/__init__.py @@ -0,0 +1,50 @@ +"""Scoring module for word segment scoring.""" + +from himotoki.scoring.caches import ( + LRUCache, + _CONJ_DATA_CACHE, + _POS_SEQ_CACHE, + _UK_CACHE, + _WORD_CACHE, + _ENTRY_CACHE, + _ARCHAIC_CACHE, + clear_scoring_caches, + preload_scoring_caches, + get_cached_entry, + build_archaic_cache, + is_arch, + is_prefer_kana, + get_non_arch_posi, +) +from himotoki.scoring.calc_score import ( + MAX_WORD_LENGTH, + SCORE_CUTOFF, + GAP_PENALTY, + LENGTH_COEFF_SEQUENCES, + IDENTICAL_WORD_SCORE_CUTOFF, + SKIP_WORDS, + COPULAE, + NO_KANJI_BREAK_PENALTY, + FINAL_PRT, + SEMI_FINAL_PRT, + NON_FINAL_PRT, + SEQ_SCORE_BONUS, + length_multiplier, + length_multiplier_coeff, + matches_conj_form, + skip_by_conj_data, + is_weak_conj_form, + compare_common, + kanji_break_penalty, + calc_score, + determine_primary_full, + get_original_text_data, + _get_original_text_data_recursive, + apply_score_mod, + get_entry_posi, + determine_primary, + get_original_text_common, + cull_segments, + gen_score, + gap_penalty, +) diff --git a/himotoki/scoring/caches.py b/himotoki/scoring/caches.py new file mode 100644 index 0000000..9fd93ba --- /dev/null +++ b/himotoki/scoring/caches.py @@ -0,0 +1,363 @@ +""" +LRU caches and cache management for scoring. +""" + +from collections import OrderedDict +from typing import Optional, List, Dict, Set + +from sqlalchemy import select, and_, func, case +from sqlalchemy.orm import Session + +from himotoki.db.models import ( + Entry, Sense, SenseProp, Conjugation, +) + +class LRUCache: + """ + A simple LRU (Least Recently Used) cache with a maximum size. + + Uses OrderedDict to maintain insertion/access order. When the cache + reaches capacity, the least recently used item is evicted. + + Thread-safety: This implementation is NOT thread-safe. For multi-threaded + use, wrap access with a lock. + """ + + def __init__(self, maxsize: int): + self.maxsize = maxsize + self._cache: OrderedDict = OrderedDict() + + def get(self, key, default=None): + """Get item, moving it to the end (most recently used).""" + if key in self._cache: + self._cache.move_to_end(key) + return self._cache[key] + return default + + def __contains__(self, key): + return key in self._cache + + def __getitem__(self, key): + self._cache.move_to_end(key) + return self._cache[key] + + def __setitem__(self, key, value): + if key in self._cache: + self._cache.move_to_end(key) + else: + if len(self._cache) >= self.maxsize: + # Evict oldest (first) item + self._cache.popitem(last=False) + self._cache[key] = value + + def __len__(self): + return len(self._cache) + + def keys(self): + return self._cache.keys() + + def clear(self): + self._cache.clear() + + +# Conjugation data cache - (seq, from_seq, tuple(conj_ids), tuple(texts)) -> List[ConjData] +_CONJ_DATA_CACHE: LRUCache = LRUCache(maxsize=2048) + +# POS cache - seq -> frozenset(posi) for per-seq caching +_POS_SEQ_CACHE: LRUCache = LRUCache(maxsize=4096) + +# UK (prefer kana) cache - frozenset(seqs) -> bool +_UK_CACHE: LRUCache = LRUCache(maxsize=1024) + +# Word lookup cache - (db_id, word, root_only) -> List[WordMatch] +_WORD_CACHE: LRUCache = LRUCache(maxsize=4096) + +# Entry cache - seq -> Entry (reduces individual lookups) +_ENTRY_CACHE: LRUCache = LRUCache(maxsize=4096) + +# Archaic words cache - populated on first use +_ARCHAIC_CACHE: Optional[Set[int]] = None + + +def clear_scoring_caches() -> None: + """ + Clear all scoring/lookup LRU caches and the archaic set. + + Call on shutdown or when switching database paths so ORM objects and + seq sets from a previous DB cannot leak into a new session. + """ + global _ARCHAIC_CACHE + _CONJ_DATA_CACHE.clear() + _POS_SEQ_CACHE.clear() + _UK_CACHE.clear() + _WORD_CACHE.clear() + _ENTRY_CACHE.clear() + _ARCHAIC_CACHE = None + + +def preload_scoring_caches(session: Session, seqs: Set[int]) -> None: + """ + Batch preload all caches needed for calc_score. + + This dramatically reduces cold-start latency by: + 1. Batch loading Entry objects + 2. Batch preloading UK (prefer-kana) status + 3. Batch preloading POS tags + + Call this before scoring a batch of segments. + + Args: + session: Database session + seqs: Set of seq numbers to preload + """ + global _ENTRY_CACHE, _UK_CACHE, _POS_SEQ_CACHE + + if not seqs: + return + + # Filter to seqs not already in entry cache + missing_seqs = seqs - set(_ENTRY_CACHE.keys()) + + if missing_seqs: + # Batch load entries + entries = session.execute( + select(Entry).where(Entry.seq.in_(missing_seqs)) + ).scalars().all() + + # LRUCache handles eviction automatically + for entry in entries: + _ENTRY_CACHE[entry.seq] = entry + + # Find seqs not yet in UK cache + uk_missing = {seq for seq in seqs if frozenset([seq]) not in _UK_CACHE} + + if uk_missing: + # Single batch query for all UK statuses + uk_seqs = set(session.execute( + select(SenseProp.seq) + .where(and_( + SenseProp.seq.in_(uk_missing), + SenseProp.tag == 'misc', + SenseProp.text == 'uk' + )) + .distinct() + ).scalars().all()) + + # Cache results for all checked seqs + for seq in uk_missing: + cache_key = frozenset([seq]) + _UK_CACHE[cache_key] = seq in uk_seqs + + # Find seqs not yet in POS cache (now using _POS_SEQ_CACHE) + pos_missing = {seq for seq in seqs if seq not in _POS_SEQ_CACHE} + + if pos_missing: + # Build archaic senses subquery once + arch_misc = {'arch', 'obsc', 'rare'} + arch_senses = ( + select(SenseProp.sense_id) + .where(and_( + SenseProp.tag == 'misc', + SenseProp.text.in_(arch_misc) + )) + ) + + # Batch query for all POS tags (excluding archaic senses) + pos_results = session.execute( + select(SenseProp.seq, SenseProp.text) + .where(and_( + SenseProp.seq.in_(pos_missing), + SenseProp.tag == 'pos', + ~SenseProp.sense_id.in_(arch_senses) + )) + ).all() + + # Group results by seq + seq_to_posi: Dict[int, Set[str]] = {seq: set() for seq in pos_missing} + for seq, pos_text in pos_results: + seq_to_posi[seq].add(pos_text) + + # LRUCache handles eviction automatically + for seq, posi in seq_to_posi.items(): + _POS_SEQ_CACHE[seq] = frozenset(posi) + + +def get_cached_entry(session: Session, seq: int) -> Optional[Entry]: + """ + Get Entry from cache or database. + Uses _ENTRY_CACHE for faster repeated lookups. + """ + global _ENTRY_CACHE + + if seq in _ENTRY_CACHE: + return _ENTRY_CACHE[seq] + + entry = session.get(Entry, seq) + + # LRUCache handles eviction automatically + if entry: + _ENTRY_CACHE[seq] = entry + + return entry + + +# ============================================================================ +# Archaic Word Detection +# ============================================================================ + +def build_archaic_cache(session: Session) -> Set[int]: + """ + Build cache of archaic/obsolete/rare word seqs. + From ichiran's *is-arch-cache*. + + Words where ALL senses are marked arch/obsc/rare are considered archaic. + A word with even one non-archaic sense is NOT considered archaic. + """ + arch_misc = {'arch', 'obsc', 'rare'} + + # Find seqs where EVERY sense has an arch/obsc/rare tag + # This is the ichiran logic: + # SELECT sense.seq FROM sense + # LEFT JOIN sense_prop sp ON (... AND sp.text IN ('arch', 'obsc', 'rare')) + # GROUP BY sense.seq HAVING EVERY(sp.id IS NOT NULL) + # + # In SQLAlchemy, we do this by: + # 1. Get all (seq, sense_id) pairs with their arch tag status + # 2. Group by seq and check that all senses have the arch tag + + from sqlalchemy import func, case, literal_column + from himotoki.db.models import Sense + + # Subquery: for each sense, is it archaic? + arch_tag_subq = ( + select(SenseProp.sense_id) + .where(and_( + SenseProp.tag == 'misc', + SenseProp.text.in_(arch_misc) + )) + ) + + # Main query: find seqs where ALL senses are in arch_tag_subq + # We count total senses and archaic senses per seq, keep only where they match + query = ( + select(Sense.seq) + .group_by(Sense.seq) + .having( + func.count(Sense.id) == func.sum( + case((Sense.id.in_(arch_tag_subq), 1), else_=0) + ) + ) + ) + arch_seqs = set(session.execute(query).scalars().all()) + + # Also add conjugations derived from archaic words + if arch_seqs: + conj_query = ( + select(Conjugation.seq) + .where(Conjugation.from_seq.in_(arch_seqs)) + .distinct() + ) + conj_seqs = set(session.execute(conj_query).scalars().all()) + arch_seqs |= conj_seqs + + return arch_seqs + + +def is_arch(session: Session, seq_set: Set[int]) -> bool: + """ + Check if all seqs in seq_set are archaic/obsolete/rare. + Uses a cached set of archaic word seqs. + """ + global _ARCHAIC_CACHE + if _ARCHAIC_CACHE is None: + _ARCHAIC_CACHE = build_archaic_cache(session) + + return all(seq in _ARCHAIC_CACHE for seq in seq_set) + + +def is_prefer_kana(session: Session, seq_set: List[int]) -> bool: + """ + Check if entries have 'uk' (usually written in kana) misc tag. + Cached for performance. + """ + global _UK_CACHE + + cache_key = frozenset(seq_set) + if cache_key in _UK_CACHE: + return _UK_CACHE[cache_key] + + result = session.execute( + select(SenseProp) + .where(and_( + SenseProp.seq.in_(seq_set), + SenseProp.tag == 'misc', + SenseProp.text == 'uk' + )) + ).scalars().first() is not None + + # LRUCache handles eviction automatically + _UK_CACHE[cache_key] = result + + return result + + +def get_non_arch_posi(session: Session, seq_set: Set[int]) -> Set[str]: + """ + Get part-of-speech tags for entries, excluding archaic senses. + From ichiran's get-non-arch-posi. + + Uses per-seq caching for better hit rate when seq_sets overlap. + """ + global _POS_SEQ_CACHE + + # Check if we have all seqs cached + all_cached = all(seq in _POS_SEQ_CACHE for seq in seq_set) + + if all_cached: + # Combine cached results + result = set() + for seq in seq_set: + result |= _POS_SEQ_CACHE[seq] + return result + + # Find which seqs need to be fetched + missing_seqs = {seq for seq in seq_set if seq not in _POS_SEQ_CACHE} + + if missing_seqs: + arch_misc = {'arch', 'obsc', 'rare'} + + # Subquery to find sense_ids with archaic props + arch_senses = ( + select(SenseProp.sense_id) + .where(and_( + SenseProp.tag == 'misc', + SenseProp.text.in_(arch_misc) + )) + ) + + # Batch query for all missing seqs + results = session.execute( + select(SenseProp.seq, SenseProp.text) + .where(and_( + SenseProp.seq.in_(missing_seqs), + SenseProp.tag == 'pos', + ~SenseProp.sense_id.in_(arch_senses) + )) + ).all() + + # Group by seq + seq_posi: Dict[int, Set[str]] = {seq: set() for seq in missing_seqs} + for seq, pos_text in results: + seq_posi[seq].add(pos_text) + + # LRUCache handles eviction automatically + for seq, posi in seq_posi.items(): + _POS_SEQ_CACHE[seq] = frozenset(posi) + + # Combine results from cache + result = set() + for seq in seq_set: + if seq in _POS_SEQ_CACHE: + result |= _POS_SEQ_CACHE[seq] + + return result diff --git a/himotoki/scoring/calc_score.py b/himotoki/scoring/calc_score.py new file mode 100644 index 0000000..d2b43d9 --- /dev/null +++ b/himotoki/scoring/calc_score.py @@ -0,0 +1,993 @@ +""" +Scoring algorithm for word segments. +""" + +import math +from typing import Optional, List, Dict, Tuple, Union, Any, Set, Callable + +from sqlalchemy import select, and_ +from sqlalchemy.orm import Session + +from himotoki.db.models import ( + Entry, KanjiText, KanaText, Sense, SenseProp, ConjProp, +) +from himotoki.characters import ( + has_kanji, mora_length, count_char_class, +) +from himotoki.types import ( + ConjData, WordMatch, CompoundWord, Segment, _is_counter_text, +) +from himotoki.scoring.caches import ( + get_cached_entry, is_arch, is_prefer_kana, get_non_arch_posi, +) +from himotoki.lookup.conj_data import get_conj_data, get_word_conj_data +from himotoki.lookup.constants import SKIP_CONJ_FORMS, WEAK_CONJ_FORMS, SUPPRESS_SINGLE_TOKEN_SEQS + +# ============================================================================ +# Scoring Constants (ported from ichiran's dict-errata.lisp) +# ============================================================================ + +# Maximum word length to search for +MAX_WORD_LENGTH = 50 + +# Score cutoff - words below this score are filtered out +# This must filter out ONLY bad kana spellings, and NOT filter out any kanji spellings +SCORE_CUTOFF = 5 + +# Length coefficient sequences (from ichiran's *length-coeff-sequences*) +# Format: coefficient for mora length 1, 2, 3, 4, 5... +LENGTH_COEFF_SEQUENCES = { + 'strong': [0, 1, 8, 24, 40, 60], # Index 0 unused, 1-based + 'weak': [0, 1, 4, 9, 16, 25, 36], + 'tail': [0, 4, 9, 16, 24], + 'ltail': [0, 4, 12, 18, 24], +} + +IDENTICAL_WORD_SCORE_CUTOFF = 0.5 +GAP_PENALTY = -500 +COPULAE: Set[int] = {2089020, 1628500} # だ, です +SKIP_WORDS: Set[int] = { + 2458040, # てもいい + 2822120, # ても良い + 2013800, # ちゃう + 2108590, # とく + 2029040, # ば + 2428180, # い + 2654250, # た + 2561100, # うまいな + 2210270, # ませんか + 2210710, # ましょうか + 2257550, # ない + 2210320, # ません + 2017560, # たい + 2394890, # とる + 2194000, # であ + 2568000, # れる/られる + 2537250, # しようとする + 2760890, # 三箱 + 2831062, # てる + 2831063, # てく + 2029030, # ものの + 2568020, # せる + 900000, # たそう (custom) + 11553382, # となら (continuative of となる "to become") — always と+なら + 11566952, # となら (continuative of となる "to neighbor") — always と+なら + 2862334, # 振り出しに戻る - prefer component words (振り出し + に + 戻る) + 1881690, # 写真を撮る - prefer component words (写真 + を + 撮る) + 2860921, # 家を出る - prefer component words (家 + を + 出る) + 2862924, # 悪口(あっこう) - prefer わるぐち reading (seq 1575730) + 10551045, # 為し(なし) - prefer 無し adjective (seq 1529560) + 10456682, # 成し(なし) - prefer 無し adjective (seq 1529560) + 1465580, # 入る(いる) - prefer はいる reading (seq 1465590) + 1751450, # 精霊(しょうりょう) - prefer せいれい reading (seq 1380230) + 2657130, # 止まる(とどまる) - prefer とまる reading (seq 1310620) + 2838238, # こうぜ(校是) - obscure word, clashes with こう+ぜ (volitional+particle) + 2260200, # にい(新) - uncommon kana reading, clashes with に+い verb patterns + 2254970, # にい(兄) - uncommon kana reading, clashes with に+い verb patterns + 1774770, # から傘(からかさ) - uncommon, clashes with から(because)+傘(umbrella) + 10253059, # うける(浮く potential) - always prefer 受ける (seq 1329590) reading +} +FINAL_PRT: Set[int] = { + 2017770, # かい + 2425930, # なの + 2130430, # け っけ + 2029130, # ぞ + 2834812, # ぜ + 2718360, # がな + 2201380, # わい + 2722170, # のう + 2751630, # かいな +} + +# Semi-final particles - final, but also have other uses +# From ichiran's *semi-final-prt* (which includes *final-prt*) +SEMI_FINAL_PRT: Set[int] = FINAL_PRT | { + 2029120, # さ + 2086640, # し + 2029110, # な + 2029080, # ね + 2029100, # わ +} + +# Non-final particles - don't get final bonus +# From ichiran's *non-final-prt* +NON_FINAL_PRT: Set[int] = { + 2139720, # ん +} +NO_KANJI_BREAK_PENALTY: Set[int] = { + 1169870, # 飲む + 1198360, # 会議 + 1277450, # 好き + 2028980, # で + 1423000, # 着る + 1164690, # 一段 + 1587040, # 言う + 2827864, # なので +} +SEQ_SCORE_BONUS: Dict[int, int] = { + 10044695: 10, # でしょう (conjugated copula) over spurious 〜で + しょうか paths +} + +def length_multiplier(length: int, power: float, len_lim: int) -> float: + """ + Calculate length multiplier: len^power until len_lim, linear after. + + Args: + length: Word length in mora + power: Exponent to use + len_lim: Limit after which growth becomes linear + + Returns: + The multiplier value + """ + if length <= len_lim: + return length ** power + return length * (len_lim ** (power - 1)) + + +def length_multiplier_coeff(length: int, coeff_class: str) -> int: + """ + Get length multiplier from coefficient sequence. + + Args: + length: Word length in mora (1-based) + coeff_class: One of 'strong', 'weak', 'tail', 'ltail' + + Returns: + The coefficient value for this length + """ + coeffs = LENGTH_COEFF_SEQUENCES.get(coeff_class) + if not coeffs: + return length + + if 0 < length < len(coeffs): + return coeffs[length] + + # Linear extrapolation for lengths beyond the table + last_coeff = coeffs[-1] + last_idx = len(coeffs) - 1 + return length * (last_coeff // last_idx) if last_idx > 0 else length +def matches_conj_form(prop: ConjProp, forms: List[Tuple]) -> bool: + """ + Check if a conjugation property matches any of the weak/skip forms. + From ichiran's test-conj-prop. + + Args: + prop: ConjProp object to test + forms: List of (conj_type, neg, fml) or (pos, conj_type, neg, fml) tuples + where None means "any value matches" + + Returns: + True if prop matches any form pattern + """ + for form in forms: + if len(form) == 3: + # (conj_type, neg, fml) format + pattern = [prop.conj_type, prop.neg, prop.fml] + if all( + r is None or l == r + for l, r in zip(pattern, form) + ): + return True + elif len(form) == 4: + # (pos, conj_type, neg, fml) format + if form[0] == prop.pos: + pattern = [prop.conj_type, prop.neg, prop.fml] + if all( + r is None or l == r + for l, r in zip(pattern, form[1:]) + ): + return True + return False + + +def skip_by_conj_data(conj_data: List[ConjData]) -> bool: + """ + Check if conjugation data should be skipped entirely. + From ichiran's skip-by-conj-data. + + Returns True if ALL conjugation data matches skip patterns. + """ + if not conj_data: + return False + + return all( + cd.prop is not None and matches_conj_form(cd.prop, SKIP_CONJ_FORMS) + for cd in conj_data + ) + + +def is_weak_conj_form(conj_data: List[ConjData]) -> bool: + """ + Check if all conjugations are weak forms (don't contribute as much). + """ + if not conj_data: + return False + + return all( + cd.prop is not None and matches_conj_form(cd.prop, WEAK_CONJ_FORMS) + for cd in conj_data + ) +def compare_common(c1: Optional[int], c2: Optional[int]) -> bool: + """ + Compare two commonness values. + Lower is better, 0 is special (very common), None is worst. + + Returns True if c1 should be sorted before c2. + """ + if c2 is None: + return c1 is not None + if c2 == 0: + return c1 is not None and c1 > 0 + if c1 is not None and c1 > 0: + return c1 < c2 + return False + + +def kanji_break_penalty( + kanji_break: List[int], + score: float, + info: Optional[Dict] = None, + text: str = "", + use_length: Optional[int] = None, + score_mod: float = 0, +) -> float: + """ + Apply penalty for breaks within kanji sequences. + + Args: + kanji_break: List of positions where kanji are broken + score: Current score + info: Score info dict + text: Word text + use_length: Context length if available + score_mod: Score modifier + + Returns: + Adjusted score + """ + if not kanji_break: + return score + + # Determine break position type + end = 'both' if len(kanji_break) > 1 else ( + 'beg' if kanji_break[0] == 0 else 'end' + ) + + bonus = 0 + ratio = 2 + posi = info.get('posi', []) if info else [] + + if info: + seq_set = info.get('seq_set', set()) + + # Check for no-penalty words + if seq_set & NO_KANJI_BREAK_PENALTY: + return score + + # Check for special す break + if end == 'beg' and text.startswith('す'): + return score + + # Adjust bonus based on POS + if end == 'beg' and 'num' in posi: + bonus += 5 + elif end == 'beg' and ('suf' in posi or 'n-suf' in posi): + bonus += 10 + elif end == 'end' and 'pref' in posi: + bonus += 12 + + if score >= SCORE_CUTOFF: + return max(SCORE_CUTOFF, (score // ratio) + bonus) + return score + + +def calc_score( + session: Session, + word: Union[WordMatch, CompoundWord], + final: bool = False, + use_length: Optional[int] = None, + score_mod: float = 0, + kanji_break: Optional[List[int]] = None, +) -> Tuple[float, Dict[str, Any]]: + """ + Calculate score for a word match. + + This is the core scoring algorithm ported from ichiran's calc-score (dict.lisp lines 777-983). + + The scoring system considers: + - Word type (kanji vs kana) + - Commonness ranking + - Part of speech (particles get special handling) + - Primary/secondary reading status + - Conjugation status and type + - Word length + - Context length (use_length) + - Kanji break penalties + - Archaic word detection + + Args: + session: Database session + word: WordMatch or CompoundWord object to score + final: True if this is at the end of the text + use_length: Context length for scoring + score_mod: Score modifier (for compound words) + kanji_break: List of kanji break positions + + Returns: + Tuple of (score, info_dict) + """ + # Handle compound words by scoring the base word with compound properties + # From ichiran dict.lisp lines 782-794 + if isinstance(word, CompoundWord): + base_word = word.get_score_base() + compound_score_mod = word.score_mod + + # Both regular compounds and abbreviation compounds use the compound + # text's mora length for use_length. This creates a penalty when the + # compound is shorter than the base word (negative difference in + # length_multiplier_coeff). + # + # For abbreviations like とまず (3 mora) from とまない (4 mora): + # use_length=3, base word len=4, difference=-1 → penalty applied + compound_use_length = mora_length(word.text) + + score, info = calc_score( + session, base_word, + use_length=compound_use_length, + score_mod=compound_score_mod, + ) + + # Add compound word conjugation data + info['conj'] = get_word_conj_data(session, word) + + # Apply kanji break penalty if needed + if kanji_break: + score = kanji_break_penalty( + kanji_break, score, + info=info, text=base_word.text, + use_length=compound_use_length, score_mod=compound_score_mod + ) + + return score, info + + # Check for counter mode (CounterText objects) + ctr_mode = _is_counter_text(word) + + reading = word.reading + text = word.text + seq = word.seq + ord_val = word.ord + common = word.common + + # Fast path: Early termination for skip words (before any DB lookups) + # This avoids expensive entry lookups and conjugation data retrieval + if seq in SKIP_WORDS: + return 0, {} + + # Suppress coarse expression tokens when compositional segmentation exists. + if seq in SUPPRESS_SINGLE_TOKEN_SEQS: + return 0, {} + + # Fast path: Final particles only score at end of text + if not final and seq in FINAL_PRT: + return 0, {} + + # Get entry info - counters don't have entries + # Use cached entry lookup for performance + entry = None if ctr_mode else get_cached_entry(session, seq) + if not entry and not ctr_mode: + return 0, {} + + # Basic properties + score = 1 + prop_score = 0 + + kanji_p = word.word_type == 'kanji' + katakana_p = not kanji_p and count_char_class(text, 'katakana') > 0 + + n_kanji = count_char_class(text, 'kanji') + word_len = max(1, mora_length(text)) + + # Conjugation info - counters don't have conjugations + conj_only = False if ctr_mode else (word.conjugations is not None and word.conjugations != 'root') + root_p = ctr_mode or (not conj_only and entry and entry.root_p) + + # Get conjugation data - counters don't have conjugations + if ctr_mode: + conj_data = [] + elif conj_only and isinstance(word.conjugations, list): + conj_data = get_conj_data(session, seq, conj_ids=word.conjugations, texts=[text]) + elif not word.is_root: + conj_data = get_conj_data(session, seq, texts=[text]) + else: + conj_data = [] + + # Handle secondary conjugations (via forms) + # If this is nil, delete all secondary conjugations from conj data + secondary_conj_p = False + if conj_data: + if all(cd.via for cd in conj_data): + secondary_conj_p = True + else: + # Remove secondary conjugations from data + conj_data = [cd for cd in conj_data if not cd.via] + + conj_of = [cd.from_seq for cd in conj_data] + conj_props = [cd.prop for cd in conj_data if cd.prop] + conj_types = [cp.conj_type for cp in conj_props] + + # conj_types_p: True if not all conjugations are weak forms + # From ichiran: "weak" forms don't contribute to scoring as much + conj_types_p = ( + root_p or + use_length is not None or + not all(matches_conj_form(prop, WEAK_CONJ_FORMS) for prop in conj_props if prop) + ) + + # Get part-of-speech info + seq_set = {seq} | set(conj_of) if seq else set() + sp_seq_set = [seq] if (seq and root_p and not use_length) else list(seq_set) + + # For counters, use 'ctr' as the part of speech + if ctr_mode: + prefer_kana = False + is_arch_p = False + posi = {'ctr'} + else: + # Check for prefer kana (uk - usually written in kana) - cached + prefer_kana = is_prefer_kana(session, sp_seq_set) + + # Check if all entries are archaic + is_arch_p = is_arch(session, set(sp_seq_set)) + + # Get part-of-speech (excluding archaic senses) - cached + posi = get_non_arch_posi(session, seq_set) + + # Common properties + common_p = common is not None + common_of = common + particle_p = 'prt' in posi + semi_final_particle_p = seq in SEMI_FINAL_PRT + non_final_particle_p = seq in NON_FINAL_PRT + pronoun_p = 'pn' in posi + cop_da_p = bool(seq_set & COPULAE) + + # Length classification (ichiran lines 836-844) + # More complex logic based on various conditions + if kanji_p and not prefer_kana: + if (root_p and not conj_data) or (use_length and 13 in conj_types): + len_threshold = 2 + elif common_p and common and 0 < common < 10: + len_threshold = 2 + elif {3, 9} & set(conj_types) and not use_length: + len_threshold = 4 + else: + len_threshold = 3 + else: + if common_p and common and 0 < common < 10: + len_threshold = 2 + elif {3, 9} & set(conj_types) and not use_length: + len_threshold = 4 + else: + len_threshold = 3 + + long_p = word_len > len_threshold + + # no_common_bonus conditions + no_common_bonus = ( + particle_p or + not conj_types_p or + (not long_p and posi == {'int'}) + ) + + use_length_bonus = 0 + + # Check for skip words in conjugation chain + # Only check the full seq_set (including conjugation sources) for DIRECT word matches, + # not when scoring a base word from a compound (use_length is set for compounds). + # This allows abbreviation compounds like とどまらず to score properly even though + # their source verb (とどまる) is in SKIP_WORDS. + if use_length is None and seq_set & SKIP_WORDS: + return 0, {} + elif seq in SKIP_WORDS: + # Always skip if the direct seq is in skip words + return 0, {} + if not root_p and skip_by_conj_data(conj_data): + return 0, {} + + # Handle inherited commonness and ord from conjugation source (ichiran lines 859-870) + # This MUST happen BEFORE primary_p determination so that ord_val is correct + if conj_data and not (ord_val == 0 and common_p): + orig_texts = get_original_text_data(session, word, conj_data) + if orig_texts: + if not common_p: + conj_of_common = [c for c, o in orig_texts if c is not None] + if conj_of_common: + common = 0 + common_p = True + # Get the "best" common value + common_of = sorted(conj_of_common, key=lambda c: (c or 1000, c == 0))[0] + + # Update ord if conjugated form has lower ord + conj_of_ord = min(o for c, o in orig_texts) + if conj_of_ord < ord_val: + ord_val = conj_of_ord + + # Primary reading check - now with archaic consideration + # Pass ord_val to use corrected ord from conjugation source + primary_p = False + if not is_arch_p: + primary_p = determine_primary_full( + session, entry, word, posi, common_p, kanji_p, + root_p, conj_data, prefer_kana, conj_types_p, cop_da_p, pronoun_p, + ord_override=ord_val + ) + + # Calculate base score (ichiran lines 890-925) + if primary_p: + if long_p: + score += 10 + elif secondary_conj_p and not kanji_p: + score += 2 + elif common_p and conj_types_p: + score += 5 + elif prefer_kana or not entry or entry.n_kanji == 0: + score += 3 + else: + score += 2 + + # Particle bonus (lines 896-902) + if particle_p and (final or not semi_final_particle_p): + score += 2 + if common_p: + score += 2 + word_len + if final and not non_final_particle_p: + if primary_p: + score += 5 + elif semi_final_particle_p: + score += 2 + + # Commonness bonus (lines 903-918) + if common_p and not no_common_bonus: + if secondary_conj_p and not use_length: + common_bonus = 4 if (kanji_p and primary_p) else 2 + elif long_p or cop_da_p or (root_p and (kanji_p or (primary_p and word_len > 2))): + if common == 0: + common_bonus = 10 + elif not primary_p: + common_bonus = max(15 - (common or 0), 10) + else: + common_bonus = max(20 - (common or 0), 10) + elif kanji_p: + common_bonus = 8 + elif primary_p: + common_bonus = 4 + elif word_len > 2 or (common and 0 < common < 10): + common_bonus = 3 + else: + common_bonus = 2 + + # Reduce bonus for continuative form (conj_type 10) + if common_bonus >= 10 and 10 in conj_types: + common_bonus -= 4 + + score += common_bonus + + # Length and kanji bonuses (lines 919-926) + if long_p: + score = max(word_len, score) + + if kanji_p: + score = max(3 if is_arch_p else 5, score) + if long_p and (n_kanji > 1 or word_len > 4): + score += 2 + + # Counter mode minimum score (ichiran line 926: (when ctr-mode (setf score (max 5 score)))) + if ctr_mode: + score = max(5, score) + + # Entry-specific score adjustments + score += SEQ_SCORE_BONUS.get(seq, 0) + + # Calculate prop_score and apply length multiplier (lines 927-937) + prop_score = score + length_class = 'strong' if (kanji_p or katakana_p) else 'weak' + score = prop_score * ( + length_multiplier_coeff(word_len, length_class) + + ((n_kanji - 1) * 5 if n_kanji > 1 else 0) + ) + + # Split scoring integration (ichiran lines 927-970) + # Check for split definition and apply split scoring + # Note: counters don't use split scoring (ichiran: (unless ctr-mode ...)) + split_info = None + if not ctr_mode: + from himotoki.splits import get_split + split_result = get_split(session, word, conj_of if conj_of else None) + + if split_result: + if ':score' in split_result.modifiers: + # Direct score addition mode + score += split_result.score_bonus + split_info = ('score', split_result.score_bonus) + elif ':pscore' in split_result.modifiers: + # Proportional score modification mode + import math + new_prop_score = max(1, prop_score + split_result.score_bonus) + score = math.ceil(score * new_prop_score / prop_score) if prop_score > 0 else score + prop_score = new_prop_score + split_info = ('pscore', split_result.score_bonus) + else: + # Standard split: sum of part scores + bonus + split_score = split_result.score_bonus + part_scores = [] + for i, part in enumerate(split_result.parts): + is_last = (i == len(split_result.parts) - 1) + # Calculate adjusted use_length for final part + part_use_length = None + if is_last and use_length: + # Subtract mora lengths of preceding parts + preceding_mora = sum( + mora_length(p.text) for p in split_result.parts[:-1] + ) + part_use_length = use_length - preceding_mora + + part_score, _ = calc_score( + session, part.reading, + final=final and is_last, + use_length=part_use_length, + score_mod=score_mod if is_last else 0, + ) + part_scores.append(part_score) + split_score += part_score + + score = split_score + split_info = ('split', split_result.score_bonus, part_scores) + + # Apply use_length bonus for context + if use_length: + tail_len = use_length - word_len + tail_class = 'ltail' if (word_len > 3 and (kanji_p or katakana_p)) else 'tail' + use_length_bonus = prop_score * length_multiplier_coeff(tail_len, tail_class) + + if score_mod: + use_length_bonus += apply_score_mod(score_mod, prop_score, tail_len) + + score += use_length_bonus + + # Build info dict + info = { + 'posi': list(posi), + 'seq_set': seq_set, + 'conj': conj_data, + 'common': common_of if common_p else None, + 'score_info': [prop_score, kanji_break, use_length_bonus, split_info], + 'kpcl': [kanji_p or katakana_p, primary_p, common_p, long_p], + } + + # Apply kanji break penalty + if kanji_break: + score = kanji_break_penalty( + kanji_break, score, + info=info, text=text, + use_length=use_length, score_mod=score_mod + ) + + return score, info + + +def determine_primary_full( + session: Session, + entry: Entry, + word: WordMatch, + posi: Set[str], + common_p: bool, + kanji_p: bool, + root_p: bool, + conj_data: List[ConjData], + prefer_kana: bool, + conj_types_p: bool, + cop_da_p: bool, + pronoun_p: bool, + ord_override: Optional[int] = None, +) -> bool: + """ + Full primary reading determination from ichiran lines 872-888. + More complete than the simple version. + + Args: + ord_override: If provided, use this ord value instead of word.ord. + This is used when the ord has been corrected based on + conjugation source data. + """ + if not entry: + return True + + # Use overridden ord if provided, otherwise use word.ord + ord_val = ord_override if ord_override is not None else word.ord + + # Prefer kana and this is kana reading + if prefer_kana and conj_types_p and not kanji_p: + if not entry.primary_nokanji: + return True + # Check nokanji flag on reading + if hasattr(word.reading, 'nokanji') and word.reading.nokanji: + return True + # Additional case: common hiragana word with ord=0 should be primary + # This handles words like きれい that are commonly written in kana + # but don't have the nokanji flag on the hiragana reading + if ord_val == 0 and common_p and (word.common == 0 or (word.common is not None and word.common < 10)): + return True + + # Primary if ord=0 or copula + if ord_val == 0 or cop_da_p: + if (kanji_p or conj_types_p) and ( + (kanji_p and not prefer_kana) or + (common_p and pronoun_p) or + entry.n_kanji == 0 + ): + return True + + # Special case: prefer_kana with kanji, ord=0, but uk is not for first sense + if prefer_kana and kanji_p and ord_val == 0: + # Check if uk prop is for ord=0 sense + first_sense_uk = session.execute( + select(SenseProp) + .join(Sense, SenseProp.sense_id == Sense.id) + .where(and_( + Sense.seq == entry.seq, + Sense.ord == 0, + SenseProp.tag == 'misc', + SenseProp.text == 'uk' + )) + ).scalars().first() + if not first_sense_uk: + return True + + return False + + +def get_original_text_data( + session: Session, + word: WordMatch, + conj_data: List[ConjData], +) -> List[Tuple[Optional[int], int]]: + """ + Get (common, ord) pairs from original (unconjugated) text. + + Ports ichiran's get-original-text* function from dict.lisp. + Extended from get_original_text_common to also return ord. + + For secondary conjugations (via forms), this function recursively + follows the conjugation chain to find the original source text. + + Args: + session: Database session + word: WordMatch object to get original text for + conj_data: List of ConjData objects for the word + + Returns: + List of (common, ord) tuples from the original source forms + """ + return _get_original_text_data_recursive(session, conj_data, [word.text]) + + +def _get_original_text_data_recursive( + session: Session, + conj_data: List[ConjData], + texts: List[str], +) -> List[Tuple[Optional[int], int]]: + """ + Recursive helper for get_original_text_data. + + Follows the conjugation chain through via forms to find the + ultimate source text and its properties. + + Args: + session: Database session + conj_data: List of ConjData objects + texts: List of text forms to look up + + Returns: + List of (common, ord) tuples from the original source forms + """ + result = [] + for cd in conj_data: + # Find matching source texts from src_map + src_texts = [] + for text, src_text in cd.src_map: + if text in texts: + src_texts.append(src_text) + + if not src_texts: + continue + + if cd.via is None: + # Direct conjugation - look up the source text in from_seq + for src_text in src_texts: + table = KanjiText if has_kanji(src_text) else KanaText + orig = session.execute( + select(table) + .where(and_(table.seq == cd.from_seq, table.text == src_text)) + ).scalars().first() + if orig: + result.append((orig.common, orig.ord)) + else: + # Secondary conjugation (via form) - recursively follow the chain + # Get conjugation data from via -> from_seq + via_conj_data = get_conj_data(session, cd.via, from_seq=cd.from_seq) + if via_conj_data: + result.extend(_get_original_text_data_recursive( + session, via_conj_data, src_texts + )) + + return result + + +def apply_score_mod( + score_mod: Union[int, float, callable, List], + score: float, + length: int, +) -> float: + """Apply score modifier.""" + if callable(score_mod): + return score_mod(score) + if isinstance(score_mod, list): + return sum(apply_score_mod(sm, score, length) for sm in score_mod) + return score * score_mod * length + + +def get_entry_posi(session: Session, seq_set: Set[int]) -> Set[str]: + """Get part-of-speech tags for entries.""" + query = ( + select(SenseProp.text) + .where(and_( + SenseProp.seq.in_(seq_set), + SenseProp.tag == 'pos' + )) + .distinct() + ) + results = session.execute(query).scalars().all() + return set(results) + + +def determine_primary( + session: Session, + entry: Entry, + word: WordMatch, + posi: Set[str], + common_p: bool, + kanji_p: bool, + root_p: bool, + conj_data: List[ConjData], +) -> bool: + """Determine if this is the primary reading for the entry.""" + if not entry: + return True + + # Check for uk (usually kana) preference + prefer_kana = session.execute( + select(SenseProp) + .where(and_( + SenseProp.seq == entry.seq, + SenseProp.tag == 'misc', + SenseProp.text == 'uk' + )) + ).scalars().first() is not None + + if prefer_kana and not kanji_p and word.ord == 0: + return True + + # Primary if ord=0 and kanji form or no kanji exists + if word.ord == 0: + if kanji_p or entry.n_kanji == 0: + return True + + # Check for pronoun with common reading + if common_p and 'pn' in posi and word.ord == 0: + return True + + return False + + +def get_original_text_common( + session: Session, + word: WordMatch, + conj_data: List[ConjData], +) -> Optional[int]: + """Get commonness from original (unconjugated) text.""" + for cd in conj_data: + for text, src_text in cd.src_map: + if text == word.text: + # Look up the source text + table = KanjiText if has_kanji(src_text) else KanaText + orig = session.execute( + select(table) + .where(and_(table.seq == cd.from_seq, table.text == src_text)) + ).scalars().first() + if orig and orig.common is not None: + return orig.common + return None +def cull_segments(segments: List[Segment]) -> List[Segment]: + """ + Filter segments to remove low-scoring duplicates. + + Keeps segments scoring at least IDENTICAL_WORD_SCORE_CUTOFF of the max. + Sorts by score descending, then by commonness ascending. + """ + if not segments: + return segments + + # Sort by score descending, then by commonness ascending + # Note: common=0 is the best (most common), so we need to handle it specially + # since 0 is falsy in Python + def get_common_key(s): + common = s.info.get('common') if s.info else None + if common is None: + return float('inf') + return common + + segments = sorted( + segments, + key=lambda s: ( + -s.score, # Score descending (higher is better) + get_common_key(s), # Commonness ascending (lower is better) + ) + ) + + max_score = max(s.score for s in segments) + cutoff = max_score * IDENTICAL_WORD_SCORE_CUTOFF + + # Protect suffix compounds from culling — they carry grammar chain info + # that dict entries don't have, even when their score is lower + return [s for s in segments if s.score >= cutoff + or getattr(s.word, 'is_compound', False)] + + +def gen_score( + session: Session, + segment: Segment, + final: bool = False, + kanji_break: Optional[List[int]] = None, +) -> Segment: + """Generate score for a segment.""" + score, info = calc_score( + session, segment.word, + final=final, kanji_break=kanji_break + ) + segment.score = score + # Preserve counter flag if it was set + if segment.info and segment.info.get('counter'): + info['counter'] = True + segment.info = info + return segment + + +# ============================================================================ +# Path Finding Utilities +# ============================================================================ + +def gap_penalty(start: int, end: int) -> int: + """Calculate penalty for ungapped text.""" + return (end - start) * GAP_PENALTY diff --git a/himotoki/segment.py b/himotoki/segment.py index 6dc63ce..b47d3d2 100644 --- a/himotoki/segment.py +++ b/himotoki/segment.py @@ -22,7 +22,7 @@ from himotoki.lookup import ( MAX_WORD_LENGTH, SCORE_CUTOFF, GAP_PENALTY, WordMatch, Segment, SegmentList, - find_word_full, gen_score, cull_segments, gap_penalty, + gen_score, cull_segments, gap_penalty, preload_scoring_caches, ) from himotoki.synergies import ( diff --git a/himotoki/splits.py b/himotoki/splits.py index 4617038..d9c5f4e 100644 --- a/himotoki/splits.py +++ b/himotoki/splits.py @@ -1,753 +1,2 @@ -""" -Split definitions module for himotoki. -Ports ichiran's dict-split.lisp split functionality. - -Splits allow compound words to be scored as the sum of their parts. -For example: 一人で → 一人 + で -This helps correctly segment expressions that appear as single entries -in the dictionary but are better understood as compound phrases. - -Types of splits: -1. Regular splits (*split_map*): Split during scoring -2. Segment splits (*segsplit_map*): Expand one segment into several during path finding -""" - -from dataclasses import dataclass, field -from typing import Optional, List, Dict, Tuple, Union, Any, Callable, Set -from functools import lru_cache - -from sqlalchemy import select, and_, or_ -from sqlalchemy.orm import Session - -from himotoki.db.models import ( - Entry, KanjiText, KanaText, Sense, SenseProp, - Conjugation, ConjProp, -) -from himotoki.characters import is_kana, as_hiragana - - -# ============================================================================ -# Split Result Types -# ============================================================================ - -@dataclass -class SplitPart: - """A part of a split compound word.""" - reading: Any # WordMatch or similar - text: str - - def __repr__(self): - return f"" - - -@dataclass -class SplitResult: - """ - Result of splitting a word. - - Supports three scoring modes via modifiers: - - Standard (no modifier): Sum of part scores + score_bonus - - ':score': Direct score addition (score_bonus added directly) - - ':pscore': Proportional score modification (score_bonus modifies prop_score) - """ - parts: List[SplitPart] - score_bonus: int - modifiers: Set[str] = field(default_factory=set) # ':score', ':pscore', or empty for standard - - def __repr__(self): - texts = [p.text for p in self.parts] - mod_str = f", modifiers={self.modifiers}" if self.modifiers else "" - return f"" - - -# ============================================================================ -# Split Maps -# ============================================================================ - -# Main split map: seq -> split function -_split_map: Dict[int, Callable] = {} - -# Segment split map: seq -> split function (for expanding segments) -_segsplit_map: Dict[int, Callable] = {} - - -def register_split(seq: int, func: Callable, is_segsplit: bool = False): - """Register a split function for a sequence.""" - target_map = _segsplit_map if is_segsplit else _split_map - target_map[seq] = func - - -# ============================================================================ -# Split Definition Macros -# ============================================================================ - -def def_simple_split( - seq: int, - score: int, - parts: List[Tuple], - is_segsplit: bool = False, -): - """ - Define a simple split for a word. - - Args: - seq: The entry sequence number - score: Score bonus for this split - parts: List of (seq, length) or (seq, length, conjugated) tuples - is_segsplit: If True, register as segment split instead - - Each part tuple: - - seq: Sequence number(s) of the part - - length: Length of this part (None for rest of string, int for fixed) - - conjugated: If True, look up conjugated form - """ - def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - offset = 0 - result_parts = [] - - for part_def in parts: - # Parse part definition - if len(part_def) == 2: - part_seqs, part_length = part_def - conjugated = False - else: - part_seqs, part_length, conjugated = part_def - - # Ensure seqs is a list - if not isinstance(part_seqs, (list, tuple)): - part_seqs = [part_seqs] - - # Calculate actual length - if part_length is None: - # Rest of string - part_text = text[offset:] - elif callable(part_length): - actual_len = part_length(text, offset) - if actual_len is None: - return None - part_text = text[offset:offset + actual_len] - else: - part_text = text[offset:offset + part_length] - - if not part_text: - return None - - # Find the word - if conjugated: - words = find_word_conj_of(session, part_text, *part_seqs) - else: - words = find_word_seq(session, part_text, *part_seqs) - - if not words: - return None - - result_parts.append(SplitPart(reading=words[0], text=part_text)) - - if part_length is not None: - if callable(part_length): - offset += part_length(text, offset) - else: - offset += part_length - - return SplitResult(parts=result_parts, score_bonus=score) - - register_split(seq, split_fn, is_segsplit) - return split_fn - - -def def_de_split(seq: int, seq_a: int, score: int = 20): - """ - Define a で split: word that ends with で. - Common pattern: X + で where X is the main meaning. - """ - def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - if not text.endswith('で'): - return None - - main_text = text[:-1] - de_text = 'で' - - # Find main part - main_words = find_word_seq(session, main_text, seq_a) - if not main_words: - return None - - # Find で particle - de_words = find_word_seq(session, de_text, 2028980) - if not de_words: - return None - - return SplitResult( - parts=[ - SplitPart(reading=main_words[0], text=main_text), - SplitPart(reading=de_words[0], text=de_text), - ], - score_bonus=score - ) - - register_split(seq, split_fn) - return split_fn - - -def def_toori_split(seq: int, seq_a: int, score: int = 50, seq_b: int = 1432930): - """ - Define a 通り split: word + 通り. - """ - def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - word_type = getattr(reading, 'word_type', 'kana') - - # Only for kanji readings - if word_type != 'kanji': - return None - - if len(text) < 3: - return None - - main_text = text[:-2] - toori_text = text[-2:] - - # Find main part - main_words = find_word_seq(session, main_text, seq_a) - if not main_words: - return None - - # Find 通り - toori_words = find_word_seq(session, toori_text, seq_b) - if not toori_words: - return None - - return SplitResult( - parts=[ - SplitPart(reading=main_words[0], text=main_text), - SplitPart(reading=toori_words[0], text=toori_text), - ], - score_bonus=score - ) - - register_split(seq, split_fn) - return split_fn - - -# ============================================================================ -# Helper Functions -# ============================================================================ - -def find_word_seq(session: Session, text: str, *seqs: int) -> List[Any]: - """Find word by text that matches specific sequences.""" - from himotoki.lookup import WordMatch - - table = KanaText if is_kana(text) else KanjiText - - results = session.execute( - select(table).where(and_( - table.text == text, - table.seq.in_(seqs) - )) - ).scalars().all() - - return [WordMatch(reading=r) for r in results] - - -def find_word_conj_of(session: Session, text: str, *seqs: int) -> List[Any]: - """Find word that is conjugation of specific sequences.""" - from himotoki.lookup import WordMatch - - table = KanaText if is_kana(text) else KanjiText - - # Direct match - direct = session.execute( - select(table).where(and_( - table.text == text, - table.seq.in_(seqs) - )) - ).scalars().all() - - # Conjugation match - indirect = session.execute( - select(table) - .join(Conjugation, table.seq == Conjugation.seq) - .where(and_( - table.text == text, - Conjugation.from_seq.in_(seqs) - )) - ).scalars().all() - - return [WordMatch(reading=r) for r in direct + indirect] - - -# ============================================================================ -# Split Lookup Functions -# ============================================================================ - -def get_split( - session: Session, - reading: Any, - conj_of: Optional[List[int]] = None, -) -> Optional[SplitResult]: - """ - Get split for a reading if one exists. - - Args: - session: Database session - reading: The reading to split - conj_of: List of sequence numbers this is conjugated from - - Returns: - SplitResult if split found, None otherwise - """ - seq = getattr(reading, 'seq', None) - if seq is None: - return None - - # Try direct match - split_fn = _split_map.get(seq) - if split_fn: - result = split_fn(session, reading) - if result and all(p.reading for p in result.parts): - return result - - # Try conjugation sources - if conj_of: - for cseq in conj_of: - split_fn = _split_map.get(cseq) - if split_fn: - result = split_fn(session, reading) - if result and all(p.reading for p in result.parts): - return result - - return None - - -def get_segsplit(session: Session, segment: Any) -> Optional[Any]: - """ - Get segment split for expanding a segment. - - Args: - session: Database session - segment: The segment to expand - - Returns: - New segment with expanded compound, or None - """ - from himotoki.lookup import Segment - - word = segment.word - seq = getattr(word, 'seq', None) - if seq is None: - return None - - # Get sequence set from info - seq_set = segment.info.get('seq_set', []) - conj_of = seq_set[1:] if len(seq_set) > 1 else [] - - # Try direct match - split_fn = _segsplit_map.get(seq) - if split_fn: - result = split_fn(session, word) - if result and all(p.reading for p in result.parts): - return _create_split_segment(segment, result) - - # Try conjugation sources - for cseq in conj_of: - split_fn = _segsplit_map.get(cseq) - if split_fn: - result = split_fn(session, word) - if result and all(p.reading for p in result.parts): - return _create_split_segment(segment, result) - - return None - - -def _create_split_segment(original_segment: Any, split_result: SplitResult) -> Any: - """Create a new segment from a split result.""" - from himotoki.lookup import Segment, CompoundWord - - # Create compound word from parts - parts = [p.reading for p in split_result.parts] - text = ''.join(p.text for p in split_result.parts) - - # Build kana with spaces - kana_parts = [] - for p in split_result.parts: - r = p.reading - if hasattr(r, 'reading'): - r = r.reading - kana_parts.append(getattr(r, 'text', p.text)) - - compound = CompoundWord( - primary=parts[0], - words=parts, - text=text, - kana=' '.join(kana_parts), - score_mod=split_result.score_bonus, - ) - - # Create new segment - new_seg = Segment( - start=original_segment.start, - end=original_segment.end, - word=compound, - text=text, - score=original_segment.score + split_result.score_bonus, - info=dict(original_segment.info), - ) - - return new_seg - - -# ============================================================================ -# Split Definitions - Port from dict-split.lisp -# ============================================================================ - -def init_splits(): - """Initialize all split definitions.""" - global _split_map, _segsplit_map - - if _split_map: - return # Already initialized - - # ====================== - # -de expressions - # ====================== - def_de_split(1163700, 1576150) # 一人で - def_de_split(1611020, 1577100) # 何で - def_de_split(1004800, 1628530) # これで - def_de_split(2810720, 1004820) # 此れまでで - def_de_split(1006840, 1006880) # その上で - def_de_split(1530610, 1530600) # 無断で - def_de_split(1245390, 1245290) # 空で - def_de_split(2719270, 1445430) # 土足で - def_de_split(1189420, 2416780) # 何用で - def_de_split(1272220, 1592990) # 交代で - def_de_split(1311360, 1311350) # 私費で - def_de_split(1368500, 1368490) # 人前で - def_de_split(1395670, 1395660) # 全体で - def_de_split(1417790, 1417780) # 単独で - def_de_split(1454270, 1454260) # 道理で - def_de_split(1479100, 1679020) # 半眼で - def_de_split(1510140, 1680900) # 別封で - def_de_split(1518550, 1529560) # 無しで - def_de_split(1531420, 1531410) # 名義で - def_de_split(1597400, 1585205) # 力尽くで - def_de_split(1679990, 2582460) # 抜き足で - def_de_split(1682060, 2085340) # 金ずくで - def_de_split(1736650, 1611710) # 水入らずで - def_de_split(1865020, 1590150) # 陰で - def_de_split(1878880, 2423450) # 差しで - def_de_split(2126220, 1802920) # 捩じり鉢巻きで - def_de_split(2136520, 2005870) # もう少しで - def_de_split(2513590, 2513650) # 詰め開きで - def_de_split(2771850, 2563780) # 気にしないで - def_de_split(2810800, 1587590) # 今までで - def_de_split(1343110, 1343100) # ところで - def_de_split(1270210, 1001640) # お陰で - - # ====================== - # -通り expressions - # ====================== - def_toori_split(1260990, 1260670) # 元通り - def_toori_split(1414570, 2082450) # 大通り - def_toori_split(1424950, 1620400) # 中通り (ちゅう通り) - def_toori_split(1424960, 1423310) # 中通り (なか通り) - def_toori_split(1820790, 1250090) # 型通り - def_toori_split(1489800, 1489340) # 表通り - def_toori_split(1523010, 1522150) # 本通り - def_toori_split(1808080, 1604890) # 目通り - def_toori_split(1368820, 1580640) # 人通り - def_toori_split(1550490, 1550190) # 裏通り - def_toori_split(1619440, 2069220) # 素通り - def_toori_split(1164910, 2821500, seq_b=1432920) # 一通り - def_toori_split(1462720, 1461140, seq_b=1432920) # 二通り - - # ====================== - # ど- prefix splits - # ====================== - _register_do_split(2142710, 1185200) # ど下手 - _register_do_split(2803190, 1595630) # どすけべ - _register_do_split(2142680, 1290210) # ど根性 - _register_do_split(2523480, 1442750) # ど田舎 - - # ====================== - # し- splits (する conjugations) - # ====================== - _register_shi_split(1005700, 1156990) # し易い - _register_shi_split(1005830, 1370760) # し吹く - _register_shi_split(1157200, 2772730) # し難い - _register_shi_split(1157220, 1195970) # し過ぎる - _register_shi_split(1157230, 1284430) # し合う - _register_shi_split(1157280, 1370090) # し尽す - _register_shi_split(1157310, 1405800) # し続ける - _register_shi_split(1304890, 1256520) # し兼ねる - _register_shi_split(1304960, 1307550) # し始める - _register_shi_split(1305110, 1338180) # し出す - _register_shi_split(1305280, 1599390) # し直す - _register_shi_split(1305290, 1212670) # し慣れる - _register_shi_split(1594300, 1596510) # し損なう - _register_shi_split(1594310, 1406680) # し損じる - _register_shi_split(1594460, 1372620) # し遂げる - _register_shi_split(1594580, 1277100) # し向ける - _register_shi_split(2518250, 1332760) # し終える - _register_shi_split(1157240, 1600260) # し残す - _register_shi_split(1304820, 1207610) # し掛ける - _register_shi_split(2858937, 1406690) # し損ねる - - # ====================== - # Complex splits - # ====================== - - # なくなる (無くなる) - def split_nakunaru(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - naku = find_word_conj_of(session, text[:2], 1529520) - naru = find_word_conj_of(session, text[2:], 1375610) - if naku and naru: - return SplitResult( - parts=[ - SplitPart(reading=naku[0], text=text[:2]), - SplitPart(reading=naru[0], text=text[2:]), - ], - score_bonus=30 - ) - return None - register_split(1529550, split_nakunaru) - - # という - def split_toiu(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - to = find_word_seq(session, 'と', 1008490) - iu = find_word_conj_of(session, text[1:], 1587040) - if to and iu: - return SplitResult( - parts=[ - SplitPart(reading=to[0], text='と'), - SplitPart(reading=iu[0], text=text[1:]), - ], - score_bonus=20 - ) - return None - register_split(1922760, split_toiu) - - # じゃない - def split_janai(session: Session, reading: Any) -> Optional[SplitResult]: - ja = find_word_seq(session, 'じゃ', 2089020) - nai = find_word_conj_of(session, 'ない', 1529520) - if ja and nai: - return SplitResult( - parts=[ - SplitPart(reading=ja[0], text='じゃ'), - SplitPart(reading=nai[0], text='ない'), - ], - score_bonus=10 - ) - return None - register_split(2755350, split_janai) - - # なら - def split_nara(session: Session, reading: Any) -> Optional[SplitResult]: - nara = find_word_conj_of(session, 'なら', 2089020) - if nara: - return SplitResult( - parts=[SplitPart(reading=nara[0], text='なら')], - score_bonus=1 - ) - return None - register_split(1009470, split_nara) - - # 気がつく - def split_kigatsuku(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - ki = find_word_seq(session, '気', 1221520) - ga = find_word_seq(session, 'が', 2028930) - tsuku = find_word_conj_of(session, text[2:], 1495740) - if ki and ga and tsuku: - return SplitResult( - parts=[ - SplitPart(reading=ki[0], text='気'), - SplitPart(reading=ga[0], text='が'), - SplitPart(reading=tsuku[0], text=text[2:]), - ], - score_bonus=100 - ) - return None - register_split(1591050, split_kigatsuku) - - # 気のせい - def split_kinosei(session: Session, reading: Any) -> Optional[SplitResult]: - ki = find_word_seq(session, '気', 1221520) - no = find_word_seq(session, 'の', 1469800) - sei = find_word_seq(session, 'せい', 1610040) - if ki and no and sei: - return SplitResult( - parts=[ - SplitPart(reading=ki[0], text='気'), - SplitPart(reading=no[0], text='の'), - SplitPart(reading=sei[0], text='せい'), - ], - score_bonus=100 - ) - return None - register_split(1221750, split_kinosei) - - # ====================== - # Segment splits - # ====================== - - # ところが - def segsplit_tokoroga(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - tokoro = find_word_seq(session, text[:-1], 1343100) - ga = find_word_seq(session, 'が', 2028930) - if tokoro and ga: - return SplitResult( - parts=[ - SplitPart(reading=tokoro[0], text=text[:-1]), - SplitPart(reading=ga[0], text='が'), - ], - score_bonus=-10 - ) - return None - register_split(1008570, segsplit_tokoroga, is_segsplit=True) - - # ところで - def segsplit_tokorode(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - tokoro = find_word_seq(session, text[:-1], 1343100) - de = find_word_seq(session, 'で', 2028980) - if tokoro and de: - return SplitResult( - parts=[ - SplitPart(reading=tokoro[0], text=text[:-1]), - SplitPart(reading=de[0], text='で'), - ], - score_bonus=-10 - ) - return None - register_split(1343110, segsplit_tokorode, is_segsplit=True) - - # とは - def segsplit_toha(session: Session, reading: Any) -> Optional[SplitResult]: - to = find_word_seq(session, 'と', 1008490) - ha = find_word_seq(session, 'は', 2028920) - if to and ha: - return SplitResult( - parts=[ - SplitPart(reading=to[0], text='と'), - SplitPart(reading=ha[0], text='は'), - ], - score_bonus=-5 - ) - return None - register_split(2028950, segsplit_toha, is_segsplit=True) - - # では - def segsplit_deha(session: Session, reading: Any) -> Optional[SplitResult]: - de = find_word_seq(session, 'で', 2028980) - ha = find_word_seq(session, 'は', 2028920) - if de and ha: - return SplitResult( - parts=[ - SplitPart(reading=de[0], text='で'), - SplitPart(reading=ha[0], text='は'), - ], - score_bonus=-5 - ) - return None - register_split(1008450, segsplit_deha, is_segsplit=True) - - # だから - def segsplit_dakara(session: Session, reading: Any) -> Optional[SplitResult]: - da = find_word_seq(session, 'だ', 2089020) - kara = find_word_seq(session, 'から', 1002980) - if da and kara: - return SplitResult( - parts=[ - SplitPart(reading=da[0], text='だ'), - SplitPart(reading=kara[0], text='から'), - ], - score_bonus=-5 - ) - return None - register_split(1007310, segsplit_dakara, is_segsplit=True) - - -def _register_do_split(seq: int, seq_b: int, score: int = 30): - """Register a ど- prefix split.""" - def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - do = find_word_seq(session, 'ど', 2252690) - rest = find_word_seq(session, text[1:], seq_b) - if do and rest: - return SplitResult( - parts=[ - SplitPart(reading=do[0], text='ど'), - SplitPart(reading=rest[0], text=text[1:]), - ], - score_bonus=score - ) - return None - register_split(seq, split_fn) - - -def _register_shi_split(seq: int, seq_b: int, score: int = 30): - """Register a し- (する conjugation) split.""" - def split_fn(session: Session, reading: Any) -> Optional[SplitResult]: - text = getattr(reading, 'text', str(reading)) - shi = find_word_conj_of(session, 'し', 1157170) - rest = find_word_conj_of(session, text[1:], seq_b) - if shi and rest: - return SplitResult( - parts=[ - SplitPart(reading=shi[0], text='し'), - SplitPart(reading=rest[0], text=text[1:]), - ], - score_bonus=score - ) - return None - register_split(seq, split_fn) - - -# Initialize on module load -init_splits() - - -# ============================================================================ -# Compound Word Support -# ============================================================================ - -@dataclass -class CompoundWord: - """ - A compound word formed by multiple parts. - Used for split and suffix compound words. - """ - primary: Any # Primary word (first part or designated primary) - words: List[Any] # All parts - text: str # Full text - kana: str # Kana reading (may have spaces) - score_mod: int = 0 # Score modifier - - @property - def seq(self) -> List[int]: - """Return seq as list for compound words.""" - seqs = [] - for w in self.words: - if hasattr(w, 'seq'): - seqs.append(w.seq) - return seqs - - @property - def word_type(self) -> str: - """Return the word type of the primary word.""" - if hasattr(self.primary, 'word_type'): - return self.primary.word_type - return 'kana' - - @property - def common(self) -> Optional[int]: - """Return commonness from primary word.""" - if hasattr(self.primary, 'common'): - return self.primary.common - return None +"""Backward-compatible shim for himotoki.splits.""" +from himotoki.grammar.splits import * # noqa: F401,F403 diff --git a/himotoki/suffixes.py b/himotoki/suffixes.py index 8db9179..bf8bc77 100644 --- a/himotoki/suffixes.py +++ b/himotoki/suffixes.py @@ -1,1753 +1,2 @@ -""" -Suffix handling module for himotoki. -Ports ichiran's dict-grammar.lisp suffix functionality. - -Suffixes are grammatical endings that attach to word stems to form compound words. -Examples include: -- たい (tai): want to... -- ている (teiru): indicates continuing action -- そう (sou): looks like... -- たくない (takunai): don't want to... - -The suffix cache maps suffix strings to (keyword, kana_form) pairs. -This enables efficient lookup during segmentation. -""" - -from dataclasses import dataclass, field -from typing import Optional, List, Dict, Tuple, Union, Any, Callable, Set -from functools import lru_cache -import threading - -from sqlalchemy import select, and_, or_ -from sqlalchemy.orm import Session - -from himotoki.db.models import ( - Entry, KanjiText, KanaText, Sense, SenseProp, - Conjugation, ConjProp, ConjSourceReading, -) -from himotoki.characters import is_kana, as_hiragana -from himotoki.constants import ( - CONJ_ADVERBIAL, CONJ_ADJECTIVE_STEM, - # Particles - SEQ_O_PREFIX, SEQ_DE, SEQ_KA, SEQ_NI, SEQ_WO, SEQ_NO, SEQ_TTE, SEQ_KARA, - SEQ_WA, SEQ_MO, - # Common verbs - SEQ_IRU, SEQ_KURU, SEQ_SURU, SEQ_TOMU, SEQ_ORU, SEQ_ARU, SEQ_OKU, SEQ_IKU, - SEQ_SHIMAU, SEQ_MORAU, SEQ_ITADAKU, SEQ_KURERU, SEQ_MIRU, SEQ_AGERU, - SEQ_HOSHII, - # Honorific/humble verb forms - SEQ_ITASU, SEQ_SARERU, SEQ_SASERU, SEQ_TOKU, - # Suffix-related - SEQ_CHAU, SEQ_CHIMAU, SEQ_TAI, SEQ_NIKUI, SEQ_II, SEQ_KUDASAI, - SEQ_SOU, SEQ_SOU_NI_NAI, SEQ_MOII, SEQ_NAGARA, - SEQ_PPOI, SEQ_GATAI, SEQ_DASU, SEQ_KIRU, SEQ_KATA, SEQ_MI, - SEQ_YASUI, SEQ_MAKURU, SEQ_NAOSU, SEQ_SOKONAU, SEQ_WASURERU, - SEQ_OERU, SEQ_ZURAI, SEQ_GIMI, SEQ_PPANASHI, SEQ_TACHI, - SEQ_AU, SEQ_KOMU, SEQ_HOUDAI, SEQ_OWARU, SEQ_HAJIMERU, SEQ_TSUKERU, - SEQ_YARU, SEQ_MAIRU, SEQ_KUDASARU, SEQ_SASHIAGERU, - # Blocked seqs - BLOCKED_NAI_SEQS, BLOCKED_NAI_X_SEQS, - # Suffix descriptions (to merge with local ones) - SUFFIX_DESCRIPTION as SUFFIX_DESCRIPTION_SEQS, -) - - -# ============================================================================ -# Suffix Descriptions -# ============================================================================ - -SUFFIX_DESCRIPTION: Dict[Union[str, int], str] = { - # Verbal suffixes - 'chau': 'indicates completion / to do something by accident or regret', - 'ha': 'topic marker particle', - 'tai': 'want to... / would like to...', - 'iru': 'indicates continuing action (to be ...ing)', - 'oru': 'indicates continuing action (to be ...ing) (humble)', - 'aru': 'indicates completion / finished action', - 'kuru': 'indicates action that had been continuing up till now / came to be', - 'oku': 'to do in advance / to leave in the current state expecting a later change', - 'kureru': '(asking) to do something for one', - 'morau': '(asking) to get somebody to do something', - 'itadaku': '(asking) to get somebody to do something (polite)', - 'iku': 'is becoming / action starting now and continuing', - 'miru': 'to try doing ...', - 'ageru': 'to do something for someone', - 'hoshii': 'want someone to ...', - 'suru': 'makes a verb from a noun', - 'itasu': 'makes a verb from a noun (humble)', - 'sareru': 'makes a verb from a noun (honorific or passive)', - 'saseru': 'let/make someone/something do ...', - 'rou': 'probably / it seems that... / I guess ...', - 'ii': "it's ok if ... / is it ok if ...?", - 'mo': 'even if ...', - 'sugiru': 'to be too (much) ...', - 'nikui': 'difficult to...', - 'sa': '-ness (degree or condition of adjective)', - 'tsutsu': 'while ... / in the process of ...', - 'nagara': 'while doing ... / simultaneously ...', - 'tsutsuaru': 'to be doing ... / to be in the process of doing ...', - 'tsuzukeru': 'to continue ...', - 'uru': 'can ... / to be able to ...', - 'sou': 'looking like ... / seeming ...', - 'nai': 'negative suffix', - 'naide': 'without doing ... / don\'t', - 'ra': 'pluralizing suffix (not polite)', - 'kudasai': 'please do ...', - 'yagaru': 'indicates disdain or contempt', - 'naru': 'to become ...', - 'desu': 'formal copula', - 'desho': "it seems/perhaps/don't you think?", - 'tosuru': 'to try to .../to be about to...', - 'garu': 'to feel .../have a ... impression of someone', - 'me': 'somewhat/-ish', - 'gai': 'worth it to ...', - 'tasou': 'seem to want to... (tai+sou)', - 'nade': 'na-adj conjunctive (and/being)', - 'nakereba': 'must do / have to (contraction)', - 'shimashou': 'let\'s ... (polite volitional contraction)', - 'kurai': 'about/approximately/to the extent of', - 'ppoi': '-ish / tends to / apt to', - 'gatai': 'difficult to... / hard to... (literary)', - 'dasu': 'to burst out doing / to start suddenly', - 'kiru': 'to do completely / to finish doing', - 'kata': 'way of doing / how to ...', - 'mi': '-ness / depth of feeling (nominalization)', - 'yasui': 'easy to ... / likely to ...', - 'makuru': 'to do relentlessly / to keep doing', - 'naosu': 'to redo / to do again', - 'sokonau': 'to fail to do / to miss doing', - 'wasureru': 'to forget to do', - 'oeru': 'to finish doing', - 'zurai': 'difficult to ... / hard to ...', - 'gimi': 'tending to / -ish / -like', - 'ppanashi': 'left doing / leaving as is', - 'tachi': 'plural (people/animals)', - 'au': 'to do mutually / to do together', - 'komu': 'to do into / to do thoroughly', - 'houdai': 'as much as one likes / unlimited', - 'owaru': 'to finish doing', - 'hajimeru': 'to start doing / to begin to', - 'tsukeru': 'to be accustomed to doing', - 'yaru': 'to do for (someone, casual)', - 'mairu': 'to go/come (humble, te-form auxiliary)', - 'kudasaru': 'to kindly do for (honorific)', - 'sashiageru': 'to do for (humble, respectful)', - # Particle seqs - imported from constants and merged - **SUFFIX_DESCRIPTION_SEQS, -} - - -# ============================================================================ -# Maximum Suffix Nesting Depth -# ============================================================================ -# Limits recursion when finding nested compound words like: -# 勉強し続けている (勉強し + 続けて + いる) -# Without a limit, pathological inputs could cause infinite recursion. -MAX_SUFFIX_DEPTH = 5 - -# Thread-local storage for tracking current recursion depth -import contextvars -_current_suffix_depth: contextvars.ContextVar[int] = contextvars.ContextVar('suffix_depth', default=0) - -# ============================================================================ -# Global Suffix Cache -# ============================================================================ - -# Cache mapping suffix text to (keyword, kana_form) pairs -_suffix_cache: Dict[str, List[Tuple[str, Optional[Any]]]] = {} - -# Set of all unique 1-char suffix endings for quick filtering -# If a word doesn't end with any of these, no suffix can match -_suffix_ending_chars: Set[str] = set() - -# Mapping from seq to suffix class -_suffix_class: Dict[int, str] = {} - -# Mapping from suffix text to suffix class (for abbreviations without seq) -_suffix_text_class: Dict[str, str] = {} - -# Lock for thread-safe cache initialization -_suffix_lock = threading.Lock() - -# Flag indicating cache is initialized -_suffix_initialized = False - - -# ============================================================================ -# Suffix Cache Initialization -# ============================================================================ - -def get_suffix_description(seq: int, text: Optional[str] = None) -> Optional[str]: - """Get description for a suffix by its seq number, with text fallback.""" - if seq is not None: - suffix_class = _suffix_class.get(seq, seq) - desc = SUFFIX_DESCRIPTION.get(suffix_class) - if desc: - return desc - # Fallback: look up by text (for abbreviation suffixes without seq) - if text: - suffix_class = _suffix_text_class.get(text) - if suffix_class: - return SUFFIX_DESCRIPTION.get(suffix_class) - return None - - -def _update_cache(text: str, value: Tuple[str, Optional[Any]], join: bool = False): - """Update suffix cache with a new entry.""" - global _suffix_cache, _suffix_ending_chars - old = _suffix_cache.get(text) - if old is None: - _suffix_cache[text] = [value] - # Track the last character of this suffix for quick filtering - if text: - _suffix_ending_chars.add(text[-1]) - elif join: - _suffix_cache[text].append(value) - else: - _suffix_cache[text] = [value] - - -def get_kana_forms(session: Session, seq: int) -> List[KanaText]: - """ - Get all kana forms for an entry and its conjugations. - - Returns kana text objects for both root and conjugated forms. - """ - # Get kana texts for this seq directly - direct = session.execute( - select(KanaText).where(KanaText.seq == seq) - ).scalars().all() - - # Also get kana texts for conjugations of this seq - conj_seqs = session.execute( - select(Conjugation.seq).where(Conjugation.from_seq == seq) - ).scalars().all() - - if conj_seqs: - indirect = session.execute( - select(KanaText).where(KanaText.seq.in_(conj_seqs)) - ).scalars().all() - else: - indirect = [] - - result = [] - for kt in direct: - kt._conj_type = 'root' - result.append(kt) - - for kt in indirect: - kt._conj_type = 'conj' - result.append(kt) - - return result - - -def get_kana_form(session: Session, seq: int, text: str, conj: Optional[str] = None) -> Optional[KanaText]: - """Get a specific kana form by seq and text.""" - result = session.execute( - select(KanaText).where(and_(KanaText.seq == seq, KanaText.text == text)) - ).scalars().first() - - if result and conj: - result._conj_type = conj - - return result - - -def _load_conjs(session: Session, key: str, seq: int, suffix_class: Optional[str] = None, join: bool = False, include_kanji: bool = False): - """Load all conjugation forms into suffix cache. - - Args: - session: Database session - key: Handler key for suffix matching - seq: Entry sequence number - suffix_class: Optional suffix class (defaults to key) - join: If True, join with existing cache entries - include_kanji: If True, also load kanji text forms - """ - from himotoki.db.models import KanjiText - actual_class = suffix_class or key - for kf in get_kana_forms(session, seq): - _update_cache(kf.text, (key, kf), join=join) - _suffix_class[kf.seq] = actual_class - - # Also load kanji forms if requested - if include_kanji: - kanji_texts = session.execute( - select(KanjiText.text).where(KanjiText.seq == kf.seq) - ).scalars().all() - for kanji_text in kanji_texts: - if kanji_text != kf.text: # Don't duplicate if already kana - _update_cache(kanji_text, (key, kf), join=join) - - -def _load_kf(key: str, kf: KanaText, suffix_class: Optional[str] = None, text: Optional[str] = None, join: bool = False): - """Load a single kana form into suffix cache.""" - actual_text = text or kf.text - actual_class = suffix_class or key - _update_cache(actual_text, (key, kf), join=join) - _suffix_class[kf.seq] = actual_class - - -def _load_abbr(key: str, text: str, join: bool = False, suffix_class: Optional[str] = None): - """Load an abbreviation into suffix cache.""" - _update_cache(text, (key, None), join=join) - if suffix_class: - _suffix_text_class[text] = suffix_class - - -def init_suffixes(session: Session, blocking: bool = True, reset: bool = False): - """ - Initialize the suffix cache. - - This loads all suffix patterns into the cache for efficient lookup. - - Args: - session: Database session - blocking: If True, wait for initialization to complete - reset: If True, force re-initialization - """ - global _suffix_cache, _suffix_class, _suffix_ending_chars, _suffix_initialized - - if _suffix_initialized and not reset: - return - - with _suffix_lock: - if _suffix_initialized and not reset: - return - - _suffix_cache = {} - _suffix_class = {} - _suffix_text_class = {} - _suffix_ending_chars = set() - - # ちゃう (chau) - completion - _load_conjs(session, 'chau', SEQ_CHAU) - _load_conjs(session, 'chau', SEQ_CHIMAU) - - # は particle with ちゃ/じゃ reading - ha_kf = get_kana_form(session, SEQ_WA, 'は') - if ha_kf: - _load_kf('chau', ha_kf, suffix_class='ha', text='ちゃ') - _load_kf('chau', ha_kf, suffix_class='ha', text='じゃ') - - # たい (tai) - want to - _load_conjs(session, 'tai', SEQ_TAI) - - # たそう (tasou) - seem to want to (tai + sou) - tasou_kf = get_kana_form(session, 900000, 'たそう') # Synthetic entry - if tasou_kf: - _load_kf('tai', tasou_kf, suffix_class='tasou') - - # 難い (nikui) - difficult to - _load_conjs(session, 'ren-', SEQ_NIKUI, suffix_class='nikui') - - # おる (oru) - humble progressive - _load_conjs(session, 'te', SEQ_ORU, suffix_class='oru') - - # ある (aru) - result state - _load_conjs(session, 'te', SEQ_ARU, suffix_class='aru') - - # いる (iru) - progressive - for kf in get_kana_forms(session, SEQ_IRU): - tkf = kf.text - if len(tkf) > 1: - _update_cache(tkf, ('teiru+', kf)) - _update_cache(tkf[1:], ('teiru', kf)) - else: - _update_cache(tkf, ('teiru', kf)) - _suffix_class[kf.seq] = 'iru' - - # くる (kuru) - coming to be - _load_conjs(session, 'te', SEQ_KURU, suffix_class='kuru') - - # おく (oku) - in advance - _load_conjs(session, 'te', SEQ_OKU, suffix_class='oku') - _load_conjs(session, 'to', SEQ_TOKU, suffix_class='oku') - - # しまう (shimau) - completion (via chau) - _load_conjs(session, 'te', SEQ_SHIMAU, suffix_class='chau') - - # くれる/もらう/いただく - request forms - _load_conjs(session, 'te+space', SEQ_KURERU, suffix_class='kureru') - _load_conjs(session, 'te+space', SEQ_MORAU, suffix_class='morau') - _load_conjs(session, 'te+space', SEQ_ITADAKU, suffix_class='itadaku') - - # みる (miru) - try doing - _load_conjs(session, 'te+space', SEQ_MIRU, suffix_class='miru') - - # あげる (ageru) - do for someone - _load_conjs(session, 'te+space', SEQ_AGERU, suffix_class='ageru') - - # ほしい (hoshii) - want someone to - _load_conjs(session, 'te+space', SEQ_HOSHII, suffix_class='hoshii') - - # やる (yaru) - do for someone (casual) - _load_conjs(session, 'te+space', SEQ_YARU, suffix_class='yaru') - - # まいる (mairu) - humble go/come (te-form auxiliary) - _load_conjs(session, 'te+space', SEQ_MAIRU, suffix_class='mairu') - - # くださる (kudasaru) - kindly do for (honorific) - _load_conjs(session, 'te+space', SEQ_KUDASARU, suffix_class='kudasaru') - - # さしあげる (sashiageru) - do for (humble/respectful) - _load_conjs(session, 'te+space', SEQ_SASHIAGERU, suffix_class='sashiageru') - - # いく (iku) - going/becoming - for kf in get_kana_forms(session, SEQ_IKU): - tkf = kf.text - if tkf.startswith('い'): - _update_cache(tkf, ('te', kf)) - if len(tkf) > 1: - _update_cache(tkf[1:], ('te', kf)) - _suffix_class[kf.seq] = 'iku' - - # いい (ii) - ok if - ii_kf = get_kana_form(session, SEQ_II, 'いい') - if ii_kf: - _load_kf('teii', ii_kf, suffix_class='ii') - - # もいい - "it's ok if" (て form + もいい) - # This creates the compound ～てもいい pattern - # Try to load from database first, otherwise register directly - moii_kf = get_kana_form(session, SEQ_MOII, 'もいい') - if moii_kf: - _load_kf('teii', moii_kf, suffix_class='ii', text='もいい') - else: - # If the custom entry doesn't exist, register abbreviation for もいい - # This allows ～てもいい to be recognized - _load_abbr('teii', 'もいい', suffix_class='ii') - - # も (mo) - even if - mo_kf = get_kana_form(session, SEQ_MO, 'も') - if mo_kf: - _load_kf('te', mo_kf, suffix_class='mo') - - # ください (kudasai) - please do - kudasai_kf = get_kana_form(session, SEQ_KUDASAI, 'ください', conj='root') - if kudasai_kf: - _load_kf('kudasai', kudasai_kf) - - # する (suru) - make verb from noun - _load_conjs(session, 'suru', SEQ_SURU) - _load_conjs(session, 'suru', SEQ_ITASU, suffix_class='itasu') - _load_conjs(session, 'suru', SEQ_SARERU, suffix_class='sareru') - _load_conjs(session, 'suru', SEQ_SASERU, suffix_class='saseru') - - # そう (sou) - looks like - # Only load そう conjugations, NOT そうにない (SEQ_SOU_NI_NAI = 2141080) - # because そうにない conjugations include そうにな which incorrectly matches - # patterns like 張り裂けそうになる (should be 張り裂けそう + に + なる) - _load_conjs(session, 'sou', SEQ_SOU) - - # NOTE: だろう/でしょう are intentionally NOT loaded as absorbing suffixes. - # They should remain separate tokens (e.g., 神 + だろう, 迷惑 + でしょう + か). - - # すぎる (sugiru) - too much - _load_conjs(session, 'sugiru', 1195970) - - # さ (sa) - -ness - sa_kf = get_kana_form(session, 2029120, 'さ') - if sa_kf: - _load_kf('sa', sa_kf) - - # つつ (tsutsu) - while - tsutsu_kf = get_kana_form(session, 1008120, 'つつ') - if tsutsu_kf: - _load_kf('ren', tsutsu_kf, suffix_class='tsutsu') - _load_conjs(session, 'ren', 2027910, suffix_class='tsutsuaru') - - # ながら (nagara) - while doing - nagara_kf = get_kana_form(session, SEQ_NAGARA, 'ながら') - if nagara_kf: - _load_kf('ren', nagara_kf, suffix_class='nagara') - - # 続ける (tsuzukeru) - to continue (V-続ける pattern) - # Include kanji forms so 続けて matches alongside つづけて - _load_conjs(session, 'ren', 1405800, suffix_class='tsuzukeru', include_kanji=True) - - # っぽい (ppoi) - -ish / tends to - # Only register base form to avoid loading 100+ conjugation forms - # which include short entries like 'ぽ', 'ぽく' causing false matches - ppoi_kf = get_kana_form(session, SEQ_PPOI, 'っぽい') - if ppoi_kf: - _load_kf('ppoi', ppoi_kf) - - # がたい (gatai) - difficult to / hard to - gatai_kf = get_kana_form(session, SEQ_GATAI, 'がたい') - if gatai_kf: - _load_kf('ren', gatai_kf, suffix_class='gatai') - - # 出す (dasu) - to burst out doing / start doing - # Only register base form to avoid kana conflicts (だし, だして etc.) - dasu_kf = get_kana_form(session, SEQ_DASU, 'だす') - if dasu_kf: - _load_kf('ren', dasu_kf, suffix_class='dasu') - _load_kf('ren', dasu_kf, suffix_class='dasu', text='出す') - - # きる (kiru) - to do completely - # Only register base form to avoid short kana conflicts - kiru_kf = get_kana_form(session, SEQ_KIRU, 'きる') - if kiru_kf: - _load_kf('ren', kiru_kf, suffix_class='kiru') - _load_kf('ren', kiru_kf, suffix_class='kiru', text='切る') - - # 方 (kata) - way of doing - kata_kf = get_kana_form(session, SEQ_KATA, 'かた') - if kata_kf: - _load_kf('ren', kata_kf, suffix_class='kata') - _load_kf('ren', kata_kf, suffix_class='kata', text='方') - - # み (mi) - adjective nominalization (-ness) - mi_kf = get_kana_form(session, SEQ_MI, 'み') - if mi_kf: - _load_kf('mi', mi_kf, suffix_class='mi') - - # やすい (yasui) - easy to / likely to - yasui_kf = get_kana_form(session, SEQ_YASUI, 'やすい') - if yasui_kf: - _load_kf('ren', yasui_kf, suffix_class='yasui') - - # まくる (makuru) - to do relentlessly - makuru_kf = get_kana_form(session, SEQ_MAKURU, 'まくる') - if makuru_kf: - _load_kf('ren', makuru_kf, suffix_class='makuru') - - # なおす (naosu) - to redo / do again - naosu_kf = get_kana_form(session, SEQ_NAOSU, 'なおす') - if naosu_kf: - _load_kf('ren', naosu_kf, suffix_class='naosu') - - # そこなう (sokonau) - to fail to do - sokonau_kf = get_kana_form(session, SEQ_SOKONAU, 'そこなう') - if sokonau_kf: - _load_kf('ren', sokonau_kf, suffix_class='sokonau') - - # わすれる (wasureru) - to forget to do - wasureru_kf = get_kana_form(session, SEQ_WASURERU, 'わすれる') - if wasureru_kf: - _load_kf('ren', wasureru_kf, suffix_class='wasureru') - - # おえる (oeru) - to finish doing - oeru_kf = get_kana_form(session, SEQ_OERU, 'おえる') - if oeru_kf: - _load_kf('ren', oeru_kf, suffix_class='oeru') - - # づらい (zurai) - difficult to - zurai_kf = get_kana_form(session, SEQ_ZURAI, 'づらい') - if zurai_kf: - _load_kf('ren', zurai_kf, suffix_class='zurai') - - # ぎみ (gimi) - tending to / -ish - gimi_kf = get_kana_form(session, SEQ_GIMI, 'ぎみ') - if gimi_kf: - _load_kf('ren', gimi_kf, suffix_class='gimi') - - # っぱなし (ppanashi) - left doing - ppanashi_kf = get_kana_form(session, SEQ_PPANASHI, 'っぱなし') - if ppanashi_kf: - _load_kf('ren', ppanashi_kf, suffix_class='ppanashi') - - # たち (tachi) - plural suffix - tachi_kf = get_kana_form(session, SEQ_TACHI, 'たち') - if tachi_kf: - _load_kf('tachi', tachi_kf) - - # 合う (au) - mutual/reciprocal compound verb - au_kf = get_kana_form(session, SEQ_AU, 'あう') - if au_kf: - _load_kf('ren', au_kf, suffix_class='au') - _load_kf('ren', au_kf, suffix_class='au', text='合う') - - # 込む (komu) - into/thoroughly compound verb - komu_kf = get_kana_form(session, SEQ_KOMU, 'こむ') - if komu_kf: - _load_kf('ren', komu_kf, suffix_class='komu') - _load_kf('ren', komu_kf, suffix_class='komu', text='込む') - - # 放題 (houdai) - as much as one likes - houdai_kf = get_kana_form(session, SEQ_HOUDAI, 'ほうだい') - if houdai_kf: - _load_kf('ren', houdai_kf, suffix_class='houdai') - _load_kf('ren', houdai_kf, suffix_class='houdai', text='放題') - - # 終わる (owaru) - to finish doing - owaru_kf = get_kana_form(session, SEQ_OWARU, 'おわる') - if owaru_kf: - _load_kf('ren+', owaru_kf, suffix_class='owaru') - _load_kf('ren+', owaru_kf, suffix_class='owaru', text='終わる') - - # 始める (hajimeru) - to start doing - hajimeru_kf = get_kana_form(session, SEQ_HAJIMERU, 'はじめる') - if hajimeru_kf: - _load_kf('ren+', hajimeru_kf, suffix_class='hajimeru') - _load_kf('ren+', hajimeru_kf, suffix_class='hajimeru', text='始める') - - # つける (tsukeru) - to be accustomed to doing - tsukeru_kf = get_kana_form(session, SEQ_TSUKERU, 'つける') - if tsukeru_kf: - _load_kf('ren', tsukeru_kf, suffix_class='tsukeru') - - # うる (uru) - can - uru_kf = get_kana_form(session, 1454500, 'うる') - if uru_kf: - _load_kf('ren', uru_kf, suffix_class='uru') - - # なく (naku) - negative - naku_kf = session.execute( - select(KanaText) - .join(Conjugation, KanaText.seq == Conjugation.seq) - .where(and_(KanaText.text == 'なく', Conjugation.from_seq == 1529520)) - ).scalars().first() - if naku_kf: - _load_kf('neg', naku_kf, suffix_class='nai') - - # ないで (naide) - negative te-form "without doing" / "don't" - naide_kf = get_kana_form(session, 2258690, 'ないで') - if naide_kf: - _load_kf('neg', naide_kf, suffix_class='naide') - - # なる (naru) - become - _load_conjs(session, 'adv', 1375610, suffix_class='naru') - - # やがる (yagaru) - disdain - _load_conjs(session, 'teren', 1012740, suffix_class='yagaru') - - # ら (ra) - plural - ra_kf = get_kana_form(session, 2067770, 'ら') - if ra_kf: - _load_kf('ra', ra_kf) - - # らしい (rashii) - seems like - _load_conjs(session, 'rashii', 1013240) - - # です / でした (desu) - formal copula forms - desu_kf = get_kana_form(session, 1628500, 'です') - if desu_kf: - _load_kf('desu', desu_kf) - deshita_kf = get_kana_form(session, 10044689, 'でした') - if deshita_kf: - _load_kf('desu', deshita_kf, text='でした') - - # とする (tosuru) - try to - _load_conjs(session, 'tosuru', 2136890) - - # くらい (kurai) - about/approximately - kurai_kf = get_kana_form(session, 1154340, 'くらい') - if kurai_kf: - _load_kf('kurai', kurai_kf) - gurai_kf = get_kana_form(session, 1154340, 'ぐらい') - if gurai_kf: - _load_kf('kurai', gurai_kf) - - # がる (garu) - feel - _load_conjs(session, 'garu', 1631750) - - # がち (gachi) - tend to - gachi_kf = get_kana_form(session, 2016470, 'がち') - if gachi_kf: - _load_kf('ren', gachi_kf, suffix_class='gachi') - - # げ (ge) - -ish - ge_kf = get_kana_form(session, 2006580, 'げ') - if ge_kf: - _load_kf('iadj', ge_kf) - - # め (me) - somewhat - me_kf = get_kana_form(session, 1604890, 'め') - if me_kf: - _load_kf('iadj', me_kf, suffix_class='me') - - # がい (gai) - worth - gai_kf = get_kana_form(session, 2606690, 'がい') - if gai_kf: - _load_kf('ren-', gai_kf, suffix_class='gai') - - # Abbreviations - _load_abbr('nai', 'ねえ') - _load_abbr('nai', 'ねぇ') - _load_abbr('nai', 'ねー') - _load_abbr('nai-x', 'ず') - _load_abbr('nai-x', 'ざる') - _load_abbr('nai-x', 'ぬ') - _load_abbr('nai-n', 'ん') - - _load_abbr('nakereba', 'なきゃ', suffix_class='nakereba') - _load_abbr('nakereba', 'なくちゃ', suffix_class='nakereba') - - _load_abbr('teba', 'ちゃ', join=True) # つ - _load_abbr('reba', 'りゃ') # る - _load_abbr('keba', 'きゃ') # く - _load_abbr('geba', 'ぎゃ') # ぐ - _load_abbr('neba', 'にゃ') # ぬ - _load_abbr('beba', 'びゃ') # ぶ - _load_abbr('meba', 'みゃ') # む - _load_abbr('seba', 'しゃ') # す - - _load_abbr('shimashou', 'ましょ', suffix_class='shimashou') - _load_abbr('dewanai', 'じゃない') - - _load_abbr('ii', 'ええ') - - # な-adjective て-form: 静かで, 元気で (copula て-form) - _load_abbr('nade', 'で', suffix_class='nade') - - _suffix_initialized = True - - -def is_suffix_cache_ready() -> bool: - """Check if suffix cache is initialized.""" - return _suffix_initialized - - -def could_have_suffix(word: str) -> bool: - """ - Quick check if a word could possibly have a suffix. - - Returns False if the word doesn't end with any character that - appears at the end of known suffixes. This is a cheap O(1) filter. - - Args: - word: Word to check - - Returns: - True if a suffix could potentially match, False if definitely not - """ - if not _suffix_initialized or len(word) < 2: - return False - # Check if word ends with any character that suffixes end with - return word[-1] in _suffix_ending_chars - - -# ============================================================================ -# Suffix Lookup Functions -# ============================================================================ - -def get_suffix_map(session: Session, text: str) -> Dict[int, List[Tuple[str, str, Optional[KanaText]]]]: - """ - Create a suffix map for the given text. - - Maps end positions to list of (suffix, keyword, kana_form) tuples. - - Args: - session: Database session - text: Text to create suffix map for - - Returns: - Dictionary mapping end position to suffix entries - """ - init_suffixes(session) - - result: Dict[int, List[Tuple[str, str, Optional[KanaText]]]] = {} - - for start in range(len(text)): - for end in range(start + 1, len(text) + 1): - substr = text[start:end] - vals = _suffix_cache.get(substr) - if vals: - for keyword, kf in vals: - if end not in result: - result[end] = [] - result[end].append((substr, keyword, kf)) - - return result - - -def get_suffixes(session: Session, word: str) -> List[Tuple[str, str, Optional[KanaText]]]: - """ - Get all suffix matches for a word. - - Searches from the end of the word backwards. - - Args: - session: Database session - word: Word to find suffixes for - - Returns: - List of (suffix, keyword, kana_form) tuples - """ - init_suffixes(session) - - results = [] - for start in range(len(word) - 1, 0, -1): - substr = word[start:] - vals = _suffix_cache.get(substr) - if vals: - for keyword, kf in vals: - results.append((substr, keyword, kf)) - - return results - - -# ============================================================================ -# Suffix Matching -# ============================================================================ - -# List of suffix types where only unique matches are allowed -SUFFIX_UNIQUE_ONLY: Set[str] = { - 'ra', 'mo', 'nikui', 'gai', 'nai-n', 'dewanai', - 'eba', 'teba', 'reba', 'keba', 'geba', 'neba', 'beba', 'meba', 'seba', 'ii', -} - -# Words that should NEVER be decomposed into root+suffix compounds. -# e.g., ため should not be split into た (adj stem of たい) + め (suffix). -BLOCKED_SUFFIX_WORDS: Set[str] = { - 'ため', # "for/in order to" - not た+め - 'だめ', # "no good" - not だ+め - 'さげ', # "lowering" - not さ+げ - 'もまず', # も+まず (also+first) - not 揉む+ず (not-rub) -} - -# Abbreviation suffix types - these create compounds that should be scored -# without using the abbreviated compound's mora length. -# Matches Ichiran's def-abbr-suffix handlers which create proxy-text objects. -ABBREVIATION_SUFFIXES: Set[str] = { - 'nai', 'nai-x', 'nai-n', 'nakereba', 'shimashou', 'dewanai', - 'teba', 'reba', 'keba', 'geba', 'neba', 'beba', 'meba', 'seba', 'ii', -} - -# Stem lengths to remove from kana for abbreviation suffixes -# These correspond to the stem parameter in ichiran's def-abbr-suffix -# e.g., 2 means remove 2 kana chars from end before adding suffix -# nai-x (ず/ざる/ぬ): stem=2 removes ない → あきらめない → あきらめ + ず = あきらめず -# nakereba (なきゃ): stem=4 removes なければ → しなければ → し + なきゃ = しなきゃ -ABBREVIATION_STEMS: Dict[str, int] = { - 'nai': 2, # ねー etc - ない → ね (remove 2: ない) - 'nai-x': 2, # ず, ざる, ぬ - ない → stem (remove 2: ない) - 'nai-n': 2, # ん contraction - ない → な (remove 2: ない) - 'nakereba': 4, # なきゃ/なくちゃ - なければ → stem (remove 4: なければ) - 'shimashou': 4, # ましょ - ましょう → stem (remove 4: ましょう) - 'dewanai': 4, # じゃない - ではない → stem (remove 4: ではない) - 'teba': 2, # ちゃ - てば → stem (remove 2) - 'reba': 2, # りゃ - れば → stem (remove 2) - 'keba': 2, # きゃ - けば → stem (remove 2) - 'geba': 2, # ぎゃ - げば → stem (remove 2) - 'neba': 2, # にゃ - ねば → stem (remove 2) - 'beba': 2, # びゃ - べば → stem (remove 2) - 'meba': 2, # みゃ - めば → stem (remove 2) - 'seba': 2, # しゃ - せば → stem (remove 2) - 'ii': 2, # ええ - いい → stem (remove 2) -} - - -def match_unique(suffix_class: str, matches: List[Any]) -> bool: - """ - Check if only unique matches are allowed for this suffix class. - - Some suffix patterns should only match when no other word exists. - """ - return suffix_class in SUFFIX_UNIQUE_ONLY - - -def find_word_suffix( - session: Session, - word: str, - matches: Optional[List[Any]] = None, - suffix_map: Optional[Dict] = None, - next_end: Optional[int] = None, - depth: int = 0, -) -> List[Any]: - """ - Find suffix matches for a word. - - This tries to match grammatical suffixes at the end of the word - and return compound word objects. - - Ports ichiran's find-word-suffix from dict-grammar.lisp lines 692-706. - - Args: - session: Database session - word: Word to find suffix for - matches: Existing matches (for uniqueness checking) - suffix_map: Pre-computed suffix map (optional) - next_end: End position in source text (optional) - depth: Current recursion depth for nested compounds - - Returns: - List of compound word matches - """ - # Block known words that should never be decomposed into root+suffix - if word in BLOCKED_SUFFIX_WORDS: - return [] - - # Guard against excessive recursion using both explicit depth and context var - current_depth = _current_suffix_depth.get() - effective_depth = max(depth, current_depth) - if effective_depth >= MAX_SUFFIX_DEPTH: - return [] - - # Set context var for handlers that may call find_word_suffix recursively - token = _current_suffix_depth.set(effective_depth + 1) - - try: - from himotoki.lookup import adjoin_word, WordMatch - - init_suffixes(session) - - # Get suffixes from map or by direct lookup - if suffix_map and next_end: - suffixes = suffix_map.get(next_end, []) - else: - suffixes = get_suffixes(session, word) - - results = [] - - for suffix, keyword, kf in suffixes: - # Check uniqueness constraint - suffix_class = _suffix_class.get(kf.seq if kf else None, keyword) - if matches and match_unique(suffix_class, matches): - continue - - offset = len(word) - len(suffix) - if offset <= 0: - continue - - root = word[:offset] - - # Get suffix function based on keyword - suffix_fn = SUFFIX_HANDLERS.get(keyword) - if not suffix_fn: - continue - - # Call suffix handler to get primary words - primary_words = suffix_fn(session, root, suffix, kf) - - for pw in primary_words: - if pw is None: - continue - - # Create suffix word for adjoin - # Get conjugation IDs for the suffix word if it's a conjugated form - suffix_conj_ids = None - if kf and hasattr(kf, '_conj_type') and kf._conj_type == 'conj': - # This is a conjugated form - get the conjugation IDs - from himotoki.db.models import Conjugation - conj_query = select(Conjugation.id).where(Conjugation.seq == kf.seq) - suffix_conj_ids = list(session.execute(conj_query).scalars().all()) - - if kf: - suffix_word = WordMatch(reading=kf, conjugations=suffix_conj_ids) - else: - # Create a placeholder for the suffix (abbreviation case) - # For abbreviations like もいい, we create a minimal placeholder - # that looks like a KanaText but without a database entry - class PlaceholderReading: - def __init__(self, text): - self.text = text - self.seq = None - self.ord = 0 - self.common = None - placeholder = PlaceholderReading(suffix) - suffix_word = WordMatch(reading=placeholder) - - # Use adjoin_word to create compound (following ichiran's pattern) - # Score is determined by the suffix handler configuration - # For 'sou' suffix, use conditional scoring based on root - if keyword in ('sou', 'sou+'): - score_mod = get_sou_score(root) - else: - score_mod = SUFFIX_SCORES.get(keyword, 0) - connector = SUFFIX_CONNECTORS.get(keyword, '') - - # Check if this is an abbreviation suffix - is_abbrev = keyword in ABBREVIATION_SUFFIXES - - # Get kana for the compound - # For primary word: get kana from reading, look up if kanji - def get_word_kana(w): - from himotoki.lookup import CompoundWord - # For CompoundWord, return the compound's kana directly - if isinstance(w, CompoundWord): - return w.kana - if hasattr(w, 'reading'): - reading = w.reading - # Check if it's a kanji reading - look up kana - if hasattr(reading, 'seq') and hasattr(reading, 'text'): - # Use the reading's ord to get the matching kana - # (e.g., 食べないで ord=1 → たべないで ord=1, not たべなくて ord=0) - reading_ord = getattr(reading, 'ord', None) - if reading_ord is not None and isinstance(reading, KanjiText): - kana_result = session.execute( - select(KanaText.text) - .where(and_(KanaText.seq == reading.seq, KanaText.ord == reading_ord)) - ).scalars().first() - if kana_result: - return kana_result - # Fallback: get first kana for this seq - kana_result = session.execute( - select(KanaText.text) - .where(KanaText.seq == reading.seq) - ).scalars().first() - if kana_result: - return kana_result - # If no kana found, assume text is already kana - return reading.text - return w.text if hasattr(w, 'text') else '' - - primary_kana = get_word_kana(pw) - suffix_kana = kf.text if kf else suffix - - # For abbreviation suffixes, remove stem characters from primary kana - # This mirrors ichiran's def-abbr-suffix destem behavior - abbr_stem = ABBREVIATION_STEMS.get(keyword, 0) - if abbr_stem > 0 and len(primary_kana) > abbr_stem: - primary_kana = primary_kana[:-abbr_stem] - - # For contraction suffixes (chau, to), the handler reconstructs - # the te-form (root + て/で) for lookup. The primary word kana - # includes trailing て/で but in the surface text, that て/で - # has been contracted (ちゃ, じゃ, と, ど). Strip the trailing - # て/で from primary kana to avoid kana inflation. - # Also use surface suffix text for kana (e.g., ちゃ not は). - if keyword in ('chau', 'to'): - if primary_kana.endswith('て') or primary_kana.endswith('で'): - primary_kana = primary_kana[:-1] - suffix_kana = suffix - - # For teiru (contracted, without leading い), the surface suffix - # is shorter than kf.text (e.g., る vs いる). Use the actual - # suffix text for kana to match the contracted pronunciation. - if keyword == 'teiru' and kf and len(suffix) < len(kf.text): - suffix_kana = suffix - - # Include connector in kana (e.g., space for suru, kudasai, te+space) - compound_kana = primary_kana + connector + suffix_kana - - compound = adjoin_word( - pw, - suffix_word, - text=word, - kana=compound_kana, - score_mod=score_mod, - is_abbrev=is_abbrev, - ) - results.append(compound) - - return results - finally: - # Reset context var to previous value - _current_suffix_depth.reset(token) - - -# Suffix scores - from def-simple-suffix definitions in ichiran's dict-grammar.lisp -SUFFIX_SCORES: Dict[str, float] = { - 'tai': 5, - 'ren': 5, - 'ren+': 10, # Higher score for longer compound verb suffixes (3+ chars) - 'ren-': 0, - 'neg': 5, - 'te': 0, - 'teiru': 3, - 'teiru+': 6, - 'te+space': 3, - 'teren': 4, - 'teii': 1, - 'chau': 5, - 'to': 0, - 'suru': 5, - 'sou': 60, # Default, but overridden by get_sou_score for specific roots - 'sou+': 1, - 'adv': 1, - 'sugiru': 5, - 'sa': 2, - 'iadj': 1, - 'mi': 1, - 'garu': 0, - 'ra': 1, - 'rashii': 3, - 'ppoi': 3, - 'tachi': 3, - 'desu': 200, - 'tosuru': 3, - 'kurai': 3, - 'nai': 5, - 'kudasai': 360, - 'nade': 3, -} - -# Suffix connectors - space between root and suffix in kana -SUFFIX_CONNECTORS: Dict[str, str] = { - 'suru': ' ', - 'kudasai': ' ', - 'te+space': ' ', - 'teii': ' ', -} - - -def get_sou_score(root: str) -> float: - """ - Get the score_mod for そう suffix based on the root. - - Ports ichiran's conditional scoring for suffix-sou: - - root "から" → 40 - - root "い" → 0 - - root "出来" → 100 - - otherwise → 60 - - Additional adjustment: reduce score for short hiragana-only roots - that could be confused with verb continuative forms (e.g., いた, よさ). - This prevents incorrect splits like 言+いたそう when 言いたそう exists. - """ - from himotoki.characters import count_char_class - - if root == 'から': - return 40 - elif root == 'い': - return 0 - elif root == '出来': - return 100 - else: - # For short hiragana-only roots, reduce the score to prevent - # incorrect splits with verb continuatives - if len(root) <= 2 and count_char_class(root, 'kanji') == 0: - return 10 # Reduced from 60 to lower priority - return 60 - - -# ============================================================================ -# Suffix Handler Functions -# ============================================================================ - -def find_word_with_conj_type(session: Session, word: str, *conj_types: int) -> List[Any]: - """Import and call find_word_with_conj_type from lookup.""" - from himotoki.lookup import find_word_with_conj_type as _find_word_with_conj_type - return _find_word_with_conj_type(session, word, *conj_types) - - -def find_word_with_pos(session: Session, word: str, *posi: str) -> List[Any]: - """Find words with specific part-of-speech tags.""" - from himotoki.lookup import find_word - from himotoki.db.models import SenseProp - from sqlalchemy import select, and_ - - words = find_word(session, word) - results = [] - - for w in words: - # Check if any sense has the required POS - has_pos = session.execute( - select(SenseProp) - .where(and_( - SenseProp.seq == w.seq, - SenseProp.tag == 'pos', - SenseProp.text.in_(posi) - )) - ).scalars().first() - - if has_pos: - results.append(w) - - return results - - -def find_word_with_neg_prop(session: Session, word: str) -> List[Any]: - """Find words with negative conjugation property.""" - from himotoki.lookup import find_word_with_conj_prop - - def filter_fn(cdata): - if cdata.prop and hasattr(cdata.prop, 'neg'): - return cdata.prop.neg - return False - - return find_word_with_conj_prop(session, word, filter_fn) - - -def _handler_tai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle たい suffix - want to...""" - if root == 'い': - return [] - return find_word_with_conj_type(session, root, 13) # Continuative - - -def _handler_ren(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle generic ren'youkei (continuative) suffix.""" - return find_word_with_conj_type(session, root, 13) - - -def _handler_teren(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle te-form + ren'youkei suffix (e.g., やがる - disdain).""" - return find_word_with_conj_type(session, root, 13) # Continuative - - -def _handler_neg(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle negative stem suffix.""" - from himotoki.lookup import CONJ_NEGATIVE_STEM - return find_word_with_conj_type(session, root, 13, CONJ_NEGATIVE_STEM) - - -def _handler_chau(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """ - Handle ちゃう suffix - contracted てしまう (completion). - - Ports ichiran's suffix-chau from dict-grammar.lisp. - The suffix starts with ち or じ (contracted from て or で). - We reconstruct the te-form by: - 1. Looking at first char of suffix to determine て or で - 2. Concatenating root + て/で to form the te-form - 3. Looking up that te-form in the database - - For example: サボっちゃった - - root = "サボっ" - - suffix = "ちゃった" (first char is ち → て) - - lookup "サボっ" + "て" = "サボって" as te-form - """ - if not suffix: - return [] - - # Map first char of suffix to te/de - first_char = suffix[0] - if first_char == 'ち': - te = 'て' - elif first_char == 'じ': - te = 'で' - else: - return [] - - # Look up root + te as te-form conjugation - te_form = root + te - return find_word_with_conj_type(session, te_form, 3) - - -def _handler_to_contracted(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """ - Handle とく suffix - contracted ておく (doing in advance). - - Ports ichiran's suffix-to from dict-grammar.lisp. - The suffix starts with と or ど (contracted from て or で). - We reconstruct the te-form similarly to chau. - - For example: 置いとく - - root = "置い" - - suffix = "とく" (first char is と → て) - - lookup "置い" + "て" = "置いて" as te-form - """ - if not suffix: - return [] - - # Map first char of suffix to te/de - first_char = suffix[0] - if first_char == 'と': - te = 'て' - elif first_char == 'ど': - te = 'で' - else: - return [] - - # Look up root + te as te-form conjugation - te_form = root + te - return find_word_with_conj_type(session, te_form, 3) - - -def _handler_te(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle て form suffix.""" - if root == 'で': - return [] - if not root.endswith('て') and not root.endswith('で'): - return [] - return find_word_with_conj_type(session, root, 3) # Te-form - - -def _handler_teiru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle ている suffix.""" - if root == 'いて': - return [] - if not root.endswith('て') and not root.endswith('で'): - return [] - # First try direct database lookup for te-form - results = find_word_with_conj_type(session, root, 3) - if results: - return results - # If not found, try to find root as a compound via suffix matching - # This enables nested compounds like 勉強し続けている (勉強し + 続けて + いる) - compound_results = find_word_suffix(session, root) - return compound_results - - -def _handler_suru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle する suffix - make verb from noun.""" - return find_word_with_pos(session, root, 'vs') - - -def _handler_sou(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle そう suffix - looks like. - - そう attaches to: - 1. Verb continuative form (ren'youkei): 食べそう, 降りそう - 2. Adjective stem (without い): 美しそう, 高そう - 3. Na-adjective root: 静かそう, 元気そう - 4. Negative なさ form: 情けなさそう - """ - from himotoki.lookup import CONJ_ADJECTIVE_STEM, CONJ_ADVERBIAL - if root in ('な', 'よ', 'よさ', 'に', 'き'): - return [] - - # Check for なさ ending (negative adjective) - if root.endswith('なさ'): - root_patched = root[:-1] + 'い' - return find_word_with_neg_prop(session, root_patched) - - results = find_word_with_conj_type(session, root, 13, CONJ_ADJECTIVE_STEM, CONJ_ADVERBIAL) - # Filter out なぜる (seq 10195060 for conjugated form) which incorrectly matches なぜ + そう - # The word なぜ is the interrogative "why", not the verb "to stroke" - results = [r for r in results if getattr(r, 'seq', None) != 10195060] - - # Also check for na-adjectives (静かそう, 元気そう) - if not results: - results = find_word_with_pos(session, root, 'adj-na') - - return results - - -def _handler_sugiru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle すぎる suffix - too much. - - すぎる attaches to: - 1. Verb continuative form (ren'youkei): 食べすぎる, 飲みすぎる - 2. Adjective stem (without い): 高すぎる, 美しすぎる - 3. Na-adjective root: 静かすぎる, 元気すぎる - 4. Negative なさ form: 情けなさすぎる - """ - if root == 'い': - return [] - - # Check for なさ ending - if root.endswith('なさ') or root.endswith('無さ'): - root_patched = root[:-1] + 'い' - return find_word_with_neg_prop(session, root_patched) - - results = [] - - # Try verb continuative form (ren'youkei, conj_type=13) - results.extend(find_word_with_conj_type(session, root, 13)) - - # Try adjective stem (add い and look for adj-i) - root_i = root + 'い' - results.extend(find_word_with_pos(session, root_i, 'adj-i')) - - # Try na-adjective root: 静かすぎる, 元気すぎる - results.extend(find_word_with_pos(session, root, 'adj-na')) - - return results - - -def _handler_sa(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle さ suffix - -ness.""" - from himotoki.lookup import CONJ_ADJECTIVE_STEM - result = find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) - result.extend(find_word_with_pos(session, root, 'adj-na')) - return result - - -def _handler_rou(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle だろう suffix - probably/conjecture. - - だろう attaches to: - 1. Verb dictionary form: 食べるだろう, 行くだろう - 2. Verb past form: 食べただろう - 3. Adjective dictionary form: 高いだろう - 4. Na-adjective root: 静かだろう - 5. Negative form: 食べないだろう - """ - results = [] - - # Try dictionary form (direct word lookup) - results.extend(find_word_with_pos(session, root, 'v1')) - results.extend(find_word_with_pos(session, root, 'v5')) - results.extend(find_word_with_pos(session, root, 'adj-i')) - results.extend(find_word_with_pos(session, root, 'adj-na')) - - # Try past form (conj_type=2) - results.extend(find_word_with_conj_type(session, root, 2)) - - # Try negative form - if root.endswith('ない'): - results.extend(find_word_with_neg_prop(session, root)) - - return results - - -def _handler_adv(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle adverbial suffix.""" - from himotoki.lookup import CONJ_ADVERBIAL - return find_word_with_conj_type(session, root, CONJ_ADVERBIAL) - - -def _handler_kudasai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle ください suffix - please do.""" - if not root.endswith('て') and not root.endswith('で'): - return [] - return find_word_with_conj_type(session, root, 3) - - -def _handler_teii(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle ていい suffix - ok if.""" - if not root.endswith('て') and not root.endswith('で'): - return [] - return find_word_with_conj_type(session, root, 3) - - -def _handler_garu(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle がる suffix - feel/show signs of. - - がる attaches to adjective stems: - 1. i-adjective stem: 欲しがる (from 欲しい) - 2. tai-compound stem: 食べたがる (from 食べたい = 食べる + たい) - 3. sou-compound stem: よさそうがる - """ - from himotoki.lookup import CONJ_ADJECTIVE_STEM - if root in ('な', 'い', 'よ'): - return [] - - result = find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) - - # If no direct adjective match, check for tai compound chain - # e.g., 食べたがる: root='食べた' → root+'い' = '食べたい' → tai compound - if not result and root.endswith('た'): - tai_form = root + 'い' # Reconstruct potential たい form - tai_results = find_word_suffix(session, tai_form) - result.extend(tai_results) - - # Also check for そ ending (そう + がる) - if root.endswith('そ'): - root_patched = root[:-1] + 'う' - result.extend(find_word_with_suffix(session, root_patched, 'sou')) - - return result - - -def _handler_nade(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle na-adjective て-form (で). - - で is the conjunctive/te-form for na-adjectives via copula だ: - 静かで (being quiet; quietly and...), 元気で (being healthy), etc. - """ - return find_word_with_pos(session, root, 'adj-na') - - -def _handler_ra(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle ら suffix - plural.""" - if root.endswith('ら'): - return [] - return find_word_with_pos(session, root, 'pn') - - -def _handler_ppoi(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle っぽい suffix - -ish / tends to. - - っぽい attaches to: - 1. Verb continuative stem: 忘れ + っぽい, 飽き + っぽい - 2. Nouns: 子供 + っぽい, 大人 + っぽい, 水 + っぽい - 3. Na-adjective root: 静か + っぽい - - Note: the っ is part of the suffix, so root already has it stripped. - """ - result = find_word_with_conj_type(session, root, 13) # Verb continuative - result.extend(find_word_with_pos(session, root, 'n')) - result.extend(find_word_with_pos(session, root, 'adj-na')) - return result - - -def _handler_mi(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle み suffix - adjective nominalization (-ness). - - み converts adjectives to nouns: - - i-adj: 深い → 深み (depth), 甘い → 甘み (sweetness) - - na-adj: 静か → 静かみ (quietness) - - Root will be the adjective stem (without い for i-adj, or bare form for na-adj). - """ - from himotoki.lookup import CONJ_ADJECTIVE_STEM - result = find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) - result.extend(find_word_with_pos(session, root, 'adj-na')) - return result - - -def _handler_tachi(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle たち suffix - plural for people/animals. - - たち attaches to nouns and pronouns: - 学生たち, 子供たち, 私たち, 猫たち - """ - result = find_word_with_pos(session, root, 'n') - result.extend(find_word_with_pos(session, root, 'pn')) - return result - - -def _handler_rashii(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle らしい suffix - seems like. - - らしい attaches to: - 1. Verb/adj conjugated forms (conj_type 2) - 2. ら-ending forms (conj_type 11) - 3. Nouns: 男らしい, 春らしい, 学生らしい - 4. Na-adjective stems: 静からしい - """ - result = find_word_with_conj_type(session, root, 2) - result.extend(find_word_with_conj_type(session, root + 'ら', 11)) - # Also try noun lookup for noun+rashii patterns - result.extend(find_word_with_pos(session, root, 'n')) - result.extend(find_word_with_pos(session, root, 'adj-na')) - return result - - -def _handler_desu(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle です suffix - formal copula.""" - # Negative copula forms (e.g., 〜ないです) - if root.endswith('ない') or root.endswith('なかった'): - return find_word_with_neg_prop(session, root) - - # na-adjective + copula (e.g., 大丈夫です, 静かでした) - if len(root) < 2: - return [] - return find_word_with_pos(session, root, 'adj-na') - - -def _handler_desho(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle でしょう suffix - polite conjecture. - - でしょう attaches to: - 1. Verb dictionary form: 食べるでしょう - 2. Verb past form: 食べたでしょう - 3. Adjective dictionary form: 高いでしょう - 4. Na-adjective root: 静かでしょう - 5. Negative form: 食べないでしょう - """ - results = [] - - # Try dictionary form (direct word lookup) - results.extend(find_word_with_pos(session, root, 'v1')) - results.extend(find_word_with_pos(session, root, 'v5')) - results.extend(find_word_with_pos(session, root, 'adj-i')) - results.extend(find_word_with_pos(session, root, 'adj-na')) - - # Try past form (conj_type=2) - results.extend(find_word_with_conj_type(session, root, 2)) - - # Try negative form - if root.endswith('ない'): - results.extend(find_word_with_neg_prop(session, root)) - - return results - - -def _handler_tosuru(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle とする suffix - try to.""" - return find_word_with_conj_type(session, root, 9) # Volitional - - -def _handler_kurai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle くらい suffix - about/approximately. - - くらい attaches to: - 1. Verb dictionary form: 食べるくらい - 2. Verb past form: 食べたくらい - 3. Verb continuative form: 食べくらい (literary) - 4. Noun/pronoun: それくらい - """ - results = [] - # Try past form - results.extend(find_word_with_conj_type(session, root, 2)) - # Try continuative form - results.extend(find_word_with_conj_type(session, root, 13)) - # Try dictionary form (verbs, adjectives) - results.extend(find_word_with_pos(session, root, 'v1')) - results.extend(find_word_with_pos(session, root, 'v5')) - results.extend(find_word_with_pos(session, root, 'adj-i')) - results.extend(find_word_with_pos(session, root, 'adj-na')) - return results - - -def _handler_iadj(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle i-adjective suffix like げ, め.""" - from himotoki.lookup import CONJ_ADJECTIVE_STEM - return find_word_with_conj_type(session, root, CONJ_ADJECTIVE_STEM) - - -# Abbreviation handlers - -def _find_word_with_neg_prop_filtered( - session: Session, - word: str, - blocked_seqs: set, - allow_root: bool = False, -) -> List[Any]: - """ - Find words with negative conjugation property, excluding certain root seqs. - - Ports ichiran's filter logic from abbr-nee and abbr-n in dict-grammar.lisp: - (and (not (find (conj-data-from cdata) '(1577980 1547720))) - (conj-neg (conj-data-prop cdata))) - - Args: - session: Database session - word: Word text to search for - blocked_seqs: Set of from_seq values to exclude - allow_root: If True, also return root forms - - Returns: - List of word matches with negative conjugation, excluding blocked seqs - """ - from himotoki.lookup import find_word_with_conj_prop - - def filter_fn(cdata): - # Must be negative form - if not (cdata.prop and hasattr(cdata.prop, 'neg') and cdata.prop.neg): - return False - # Exclude blocked root seqs (居ない, 来ない create problems) - if cdata.from_seq in blocked_seqs: - return False - return True - - return find_word_with_conj_prop(session, word, filter_fn, allow_root=allow_root) - - -def _handler_abbr_nai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """ - Handle ない abbreviation (ねえ, ねぇ, ねー etc.). - - Ports ichiran's abbr-nee from dict-grammar.lisp. - Blocks いる (SEQ_IRU) and 来る (SEQ_KURU) conjugations to avoid false matches. - Allows root forms (:allow-root t in ichiran). - """ - return _find_word_with_neg_prop_filtered( - session, root + 'ない', BLOCKED_NAI_SEQS, allow_root=True - ) - - -def _handler_abbr_nai_n(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """ - Handle ん contraction (nai-n suffix). - - Ports ichiran's abbr-n from dict-grammar.lisp. - Blocks いる (SEQ_IRU) and 来る (SEQ_KURU) conjugations to avoid false matches. - Does NOT allow root forms (differs from abbr-nee). - - Example: 考えてん should NOT match 考えていないん (negative of いる). - Instead, it should match 考えて + ん separately. - """ - return _find_word_with_neg_prop_filtered( - session, root + 'ない', BLOCKED_NAI_SEQS, allow_root=False - ) - - -def _handler_abbr_nx(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """ - Handle ず/ざる/ぬ abbreviation (nai-x suffix). - - Ports ichiran's abbr-nx from dict-grammar.lisp. - Blocks する (SEQ_SURU) and 富む (SEQ_TOMU) conjugations. - Special case: せ -> しない (for する). - """ - if root == 'せ': - from himotoki.splits import find_word_conj_of - return find_word_conj_of(session, 'しない', SEQ_SURU) - - from himotoki.lookup import find_word_with_conj_prop - - def filter_fn(cdata): - # Must be negative form - if not (cdata.prop and hasattr(cdata.prop, 'neg') and cdata.prop.neg): - return False - # Exclude blocked words (する, 富む) - if cdata.from_seq in BLOCKED_NAI_X_SEQS: - return False - return True - - return find_word_with_conj_prop(session, root + 'ない', filter_fn) - - -def _handler_abbr_nakereba(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle なきゃ/なくちゃ abbreviation.""" - from himotoki.lookup import find_word_full - return find_word_full(session, root + 'なければ') - - -def _handler_abbr_shimasho(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle ましょ abbreviation (of ましょう - polite volitional).""" - from himotoki.lookup import find_word_full - return find_word_full(session, root + 'ましょう') - - -def _handler_abbr_dewanai(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle じゃない abbreviation.""" - from himotoki.lookup import find_word_full - return find_word_full(session, root + 'ではない') - - -def _handler_abbr_eba(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle conditional abbreviations (りゃ, きゃ, etc.).""" - from himotoki.lookup import find_word_full - # Map abbreviation to full conditional form - suffix_map = { - 'ちゃ': 'てば', - 'りゃ': 'れば', - 'きゃ': 'けば', - 'ぎゃ': 'げば', - 'にゃ': 'ねば', - 'びゃ': 'べば', - 'みゃ': 'めば', - 'しゃ': 'せば', - } - full_suffix = suffix_map.get(suffix) - if full_suffix: - return find_word_full(session, root + full_suffix) - return [] - - -def _handler_abbr_ii(session: Session, root: str, suffix: str, kf: Optional[KanaText]) -> List[Any]: - """Handle ええ abbreviation for いい.""" - from himotoki.lookup import find_word_full - return find_word_full(session, root + 'いい') - - -# Mapping of suffix keywords to handlers -SUFFIX_HANDLERS: Dict[str, Callable] = { - 'tai': _handler_tai, - 'ren': _handler_ren, - 'ren+': _handler_ren, - 'ren-': _handler_ren, - 'neg': _handler_neg, - 'te': _handler_te, - 'teiru': _handler_teiru, - 'teiru+': _handler_teiru, - 'te+space': _handler_te, - 'suru': _handler_suru, - 'sou': _handler_sou, - 'sou+': _handler_sou, - 'sugiru': _handler_sugiru, - 'sa': _handler_sa, - 'adv': _handler_adv, - 'kudasai': _handler_kudasai, - 'teii': _handler_teii, - 'garu': _handler_garu, - 'teren': _handler_teren, - 'ra': _handler_ra, - 'rashii': _handler_rashii, - 'desu': _handler_desu, - 'tosuru': _handler_tosuru, - 'kurai': _handler_kurai, - 'iadj': _handler_iadj, - 'mi': _handler_mi, - 'nade': _handler_nade, - 'ppoi': _handler_ppoi, - 'tachi': _handler_tachi, - 'chau': _handler_chau, # Contracted てしまう with te-reconstruction - 'to': _handler_to_contracted, # Contracted ておく with te-reconstruction - # Abbreviations - each has distinct behavior matching ichiran - 'nai': _handler_abbr_nai, # ねえ, ねぇ, ねー - allows root forms - 'nai-x': _handler_abbr_nx, # ず, ざる, ぬ - blocks する - 'nai-n': _handler_abbr_nai_n, # ん contraction - blocks いる/来る, no root - 'nakereba': _handler_abbr_nakereba, - 'shimashou': _handler_abbr_shimasho, - 'dewanai': _handler_abbr_dewanai, - 'teba': _handler_abbr_eba, - 'reba': _handler_abbr_eba, - 'keba': _handler_abbr_eba, - 'geba': _handler_abbr_eba, - 'neba': _handler_abbr_eba, - 'beba': _handler_abbr_eba, - 'meba': _handler_abbr_eba, - 'seba': _handler_abbr_eba, - 'ii': _handler_abbr_ii, -} +"""Backward-compatible shim for himotoki.suffixes.""" +from himotoki.grammar.suffixes import * # noqa: F401,F403 diff --git a/himotoki/synergies.py b/himotoki/synergies.py index 74ba44d..e1193db 100644 --- a/himotoki/synergies.py +++ b/himotoki/synergies.py @@ -1,2649 +1,4 @@ -""" -Synergies and Segfilters module for himotoki. -Ports ichiran's dict-grammar.lisp synergy and segfilter functionality. - -SYNERGIES: -Give bonuses to two consecutive words in a path when they form -common grammatical patterns. For example: -- noun + particle: 学校 + で (+10-40) -- na-adjective + な/に: 静か + な (+15) -- to-adverb + と: ゆっくり + と (+10-50) - -SEGFILTERS: -Hard constraints that ban certain word combinations that would -otherwise be grammatically invalid. For example: -- Auxiliary verbs must follow continuative form -- ない can't follow は unless it's a proper form -""" - -from dataclasses import dataclass, field -from typing import Optional, List, Dict, Tuple, Union, Any, Callable, Set -from functools import lru_cache - -from sqlalchemy import select, and_, or_ -from sqlalchemy.orm import Session - -from himotoki.db.models import KanjiText, KanaText -from himotoki.constants import ( - # Particles - SEQ_WA, SEQ_GA, SEQ_NI, SEQ_DE, SEQ_HE, SEQ_WO, SEQ_NO, SEQ_TO, SEQ_MO, SEQ_YA, SEQ_KA, - # Compound particles - SEQ_NIHA, SEQ_TOHA, SEQ_TOKA, SEQ_TOSHITE, SEQ_DESAE, - # Other particles - SEQ_DAKE, SEQ_GORO, SEQ_MADE, SEQ_NADO, SEQ_NOMI, SEQ_SAE, SEQ_TTE, SEQ_KARA, SEQ_NITOTTE, - # Verbs - SEQ_SURU, SEQ_IRU, SEQ_KURU, - # Nouns with reading ambiguity - SEQ_MAE_NOUN, SEQ_HOU_NOUN, SEQ_MEN_NOUN, - SEQ_HITO_NOUN, SEQ_NAKA_NOUN, - # Verbs with reading ambiguity - SEQ_TOMARU, SEQ_TODOMARU, - # Adjectives with reading ambiguity - SEQ_KARAI, SEQ_TSURAI, - # Expressions - SEQ_NITSURE, SEQ_OSUSUME, - # Pre-built set - NOUN_PARTICLES, -) - - -# ============================================================================ -# Filter Caching System -# ============================================================================ - -# Global filter ID counter for unique identification -_filter_id_counter = 0 - - -def _get_next_filter_id() -> int: - """Get a unique filter ID.""" - global _filter_id_counter - _filter_id_counter += 1 - return _filter_id_counter - - -def cached_filter(filter_fn: Callable) -> Callable: - """ - Wrap a filter function with segment-level caching. - - Each filter gets a unique ID, and results are cached on the segment - to avoid re-running the same filter on the same segment. - """ - filter_id = _get_next_filter_id() - - def _cached_filter(segment: Any) -> bool: - # Try to get cached result - if hasattr(segment, 'get_filter_result'): - cached = segment.get_filter_result(filter_id) - if cached is not None: - return cached - - # Compute and cache result - result = filter_fn(segment) - - if hasattr(segment, 'set_filter_result'): - segment.set_filter_result(filter_id, result) - - return result - - # Preserve the filter's identity for debugging - _cached_filter.__name__ = getattr(filter_fn, '__name__', 'cached_filter') - _cached_filter._filter_id = filter_id - - return _cached_filter - - -# ============================================================================ -# Synergy Data Structure -# ============================================================================ - -@dataclass(slots=True) -class Synergy: - """Represents a synergy between two adjacent segments.""" - description: str - connector: str # Space or empty string between words - score: int - start: int # Position where synergy starts - end: int # Position where synergy ends - - -def get_segment_score_synergy(syn: Synergy) -> int: - """Get the score value from a synergy.""" - return syn.score - - -# ============================================================================ -# Synergy List and Registration -# ============================================================================ - -_synergy_list: List[Callable] = [] - - -def register_synergy(func: Callable): - """Register a synergy function.""" - _synergy_list.append(func) - return func - - -# ============================================================================ -# Filter Helpers -# ============================================================================ - -def _filter_is_noun_impl(segment: Any) -> bool: - """Check if segment is a noun (implementation).""" - info = getattr(segment, 'info', {}) - kpcl = info.get('kpcl', [False, False, False, False]) - if len(kpcl) < 4: - kpcl = kpcl + [False] * (4 - len(kpcl)) - k, p, c, l = kpcl - - posi = info.get('posi', []) - noun_pos = {'n', 'n-adv', 'n-t', 'adj-na', 'n-suf', 'pn'} - - if (l or k or (p and c)) and noun_pos.intersection(posi): - return True - - # Check for counter - word = getattr(segment, 'word', None) - if word and hasattr(word, '__class__') and 'counter' in word.__class__.__name__.lower(): - return bool(info.get('seq_set')) - - return False - - -# Create cached version of filter_is_noun -filter_is_noun = cached_filter(_filter_is_noun_impl) - - -# Cache for filter_is_pos filters to avoid creating duplicates -_pos_filter_cache: Dict[frozenset, Callable] = {} - - -def filter_is_pos(*pos_list: str): - """Create filter for specific parts of speech (cached).""" - pos_key = frozenset(pos_list) - if pos_key in _pos_filter_cache: - return _pos_filter_cache[pos_key] - - pos_set = set(pos_list) - - def _filter(segment: Any) -> bool: - info = getattr(segment, 'info', {}) - posi = info.get('posi', []) - return bool(pos_set.intersection(posi)) - - result = cached_filter(_filter) - _pos_filter_cache[pos_key] = result - return result - - -# Cache for filter_in_seq_set filters -_seq_filter_cache: Dict[frozenset, Callable] = {} - - -def filter_in_seq_set(*seqs: int): - """Create filter for specific sequence numbers (cached).""" - seq_key = frozenset(seqs) - if seq_key in _seq_filter_cache: - return _seq_filter_cache[seq_key] - - seq_set = set(seqs) - - def _filter(segment: Any) -> bool: - info = getattr(segment, 'info', {}) - segment_seqs = info.get('seq_set', []) - return bool(seq_set.intersection(segment_seqs)) - - result = cached_filter(_filter) - _seq_filter_cache[seq_key] = result - return result - - -# Cache for filter_in_seq_set_simple filters -_seq_simple_filter_cache: Dict[frozenset, Callable] = {} - - -def filter_in_seq_set_simple(*seqs: int): - """Filter for seqs, checking that word is not compound (cached).""" - seq_key = frozenset(seqs) - if seq_key in _seq_simple_filter_cache: - return _seq_simple_filter_cache[seq_key] - - seq_set = set(seqs) - - def _filter(segment: Any) -> bool: - word = getattr(segment, 'word', None) - if not word: - return False - seq = getattr(word, 'seq', None) - if isinstance(seq, list): - return False - info = getattr(segment, 'info', {}) - segment_seqs = info.get('seq_set', []) - return bool(seq_set.intersection(segment_seqs)) - - result = cached_filter(_filter) - _seq_simple_filter_cache[seq_key] = result - return result - - -# Cache for filter_is_conjugation filters -_conj_filter_cache: Dict[int, Callable] = {} - - -def filter_is_conjugation(conj_type: int): - """Create filter for specific conjugation type (cached).""" - if conj_type in _conj_filter_cache: - return _conj_filter_cache[conj_type] - - def _filter(segment: Any) -> bool: - info = getattr(segment, 'info', {}) - conj = info.get('conj', []) - for cdata in conj: - if hasattr(cdata, 'prop') and cdata.prop: - if getattr(cdata.prop, 'conj_type', None) == conj_type: - return True - return False - - result = cached_filter(_filter) - _conj_filter_cache[conj_type] = result - return result - - -# Cache for filter_is_compound_end filters -_compound_end_filter_cache: Dict[frozenset, Callable] = {} - - -def filter_is_compound_end(*seqs: int): - """Filter for compound words ending with specific seqs (cached).""" - seq_key = frozenset(seqs) - if seq_key in _compound_end_filter_cache: - return _compound_end_filter_cache[seq_key] - - seq_set = set(seqs) - - def _filter(segment: Any) -> bool: - word = getattr(segment, 'word', None) - if not word: - return False - seq = getattr(word, 'seq', None) - if isinstance(seq, list) and seq: - return seq[-1] in seq_set - return False - - result = cached_filter(_filter) - _compound_end_filter_cache[seq_key] = result - return result - - -# Cache for filter_is_compound_end_text filters -_compound_end_text_filter_cache: Dict[frozenset, Callable] = {} - - -def filter_is_compound_end_text(*texts: str): - """Filter for compound words ending with specific texts (cached).""" - text_key = frozenset(texts) - if text_key in _compound_end_text_filter_cache: - return _compound_end_text_filter_cache[text_key] - - text_set = set(texts) - - def _filter(segment: Any) -> bool: - word = getattr(segment, 'word', None) - if not word: - return False - seq = getattr(word, 'seq', None) - if not isinstance(seq, list): - return False - words = getattr(word, 'words', []) - if words: - last_word = words[-1] - text = getattr(last_word, 'text', '') - return text in text_set - return False - - result = cached_filter(_filter) - _compound_end_text_filter_cache[text_key] = result - return result - - -def filter_short_kana(length: int, except_list: Optional[List[str]] = None): - """Filter for short kana words.""" - except_set = set(except_list) if except_list else set() - - def _filter(segment_list: Any) -> bool: - segments = getattr(segment_list, 'segments', []) - if not segments: - return False - seg = segments[0] - - seg_len = segment_list.end - segment_list.start - if seg_len > length: - return False - - info = getattr(seg, 'info', {}) - kpcl = info.get('kpcl', [False, False, False, False]) - if kpcl and kpcl[0]: # Has kanji - return False - - text = getattr(seg, 'text', '') or getattr(seg.word, 'text', '') - if text in except_set: - return False - - return True - - return _filter - - -# ============================================================================ -# Synergy Definitions -# ============================================================================ - -def def_generic_synergy( - name: str, - filter_left: Callable, - filter_right: Callable, - description: str, - score: Union[int, Callable], - connector: str = " ", -): - """ - Define a generic synergy between two segment lists. - """ - def synergy_fn(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - start = seg_list_left.end - end = seg_list_right.start - - # Must be adjacent - if start != end: - return [] - - left_segments = [s for s in seg_list_left.segments if filter_left(s)] - right_segments = [s for s in seg_list_right.segments if filter_right(s)] - - if not left_segments or not right_segments: - return [] - - # Calculate score - if callable(score): - actual_score = score(seg_list_left, seg_list_right) - else: - actual_score = score - - # Create synergy - syn = Synergy( - description=description, - connector=connector, - score=actual_score, - start=start, - end=end, - ) - - # Return modified segment lists with synergy - from himotoki.lookup import SegmentList - new_left = SegmentList( - segments=left_segments, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_segments, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_fn) - return synergy_fn - - -# Define all synergies -def _init_synergies(): - """Initialize all synergy definitions.""" - - # Helper to check for specific seq combinations that should NOT get noun+prt synergy - def not_to_before_wa(left_seg_list: Any, right_seg_list: Any) -> bool: - """Return False if left is と (SEQ_TO) and right is は (SEQ_WA).""" - left_seqs = set() - right_seqs = set() - for seg in getattr(left_seg_list, 'segments', []): - info = getattr(seg, 'info', {}) - left_seqs.update(info.get('seq_set', set())) - for seg in getattr(right_seg_list, 'segments', []): - info = getattr(seg, 'info', {}) - right_seqs.update(info.get('seq_set', set())) - # Block synergy if left has と and right has は - if SEQ_TO in left_seqs and SEQ_WA in right_seqs: - return False - return True - - # Create a modified filter_is_noun that also checks the right side - def filter_is_noun_not_to_wa(segment: Any, right_seg_list: Any = None) -> bool: - if not filter_is_noun(segment): - return False - # This will be used via custom synergy below - return True - - # noun + particle (with exclusion for と + は) - def synergy_noun_particle(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Custom noun+particle synergy that excludes と + は.""" - # Check if left is noun - left_nouns = [s for s in seg_list_left.segments if filter_is_noun(s)] - if not left_nouns: - return [] - - # Check if right is particle - particle_filter = filter_in_seq_set(*NOUN_PARTICLES) - right_particles = [s for s in seg_list_right.segments if particle_filter(s)] - if not right_particles: - return [] - - # Check serial - if seg_list_left.end != seg_list_right.start: - return [] - - # Check for と + は case that should be excluded - if not not_to_before_wa(seg_list_left, seg_list_right): - return [] - - # Create synergy - length = seg_list_right.end - seg_list_right.start - score = 10 + 4 * length - - synergy = Synergy( - description="noun+prt", - connector=" ", - score=score, - start=seg_list_left.end, - end=seg_list_right.start, - ) - - return [(seg_list_right, synergy, seg_list_left)] - - register_synergy(synergy_noun_particle) - - # noun + だ - def_generic_synergy( - name="synergy-noun-da", - filter_left=filter_is_noun, - filter_right=filter_in_seq_set(2089020), # だ - description="noun+da", - score=10, - connector=" ", - ) - - # の + だ/です/なんだ - def_generic_synergy( - name="synergy-no-da", - filter_left=filter_in_seq_set(1469800, 2139720), # の, ん - filter_right=filter_in_seq_set(2089020, 1007370, 1928670), # だ, だった, だろう - description="no da/desu", - score=15, - connector=" ", - ) - - # そう + なんだ - def_generic_synergy( - name="synergy-sou-nanda", - filter_left=filter_in_seq_set(2137720), # そう - filter_right=filter_in_seq_set(2140410), # なんだ - description="sou na n da", - score=50, - connector=" ", - ) - - # no-adjective + の - def_generic_synergy( - name="synergy-no-adjectives", - filter_left=filter_is_pos("adj-no"), - filter_right=filter_in_seq_set(1469800), # の - description="no-adjective", - score=15, - connector=" ", - ) - - # na-adjective + な/に - def_generic_synergy( - name="synergy-na-adjectives", - filter_left=filter_is_pos("adj-na"), - filter_right=filter_in_seq_set(2029110, 2028990), # な, に - description="na-adjective", - score=15, - connector=" ", - ) - - # to-adverb + と - def_generic_synergy( - name="synergy-to-adverbs", - filter_left=filter_is_pos("adv-to"), - filter_right=filter_in_seq_set(1008490), # と - description="to-adverb", - score=lambda l, r: 10 + 10 * (l.end - l.start), - connector=" ", - ) - - # noun + 中 - def_generic_synergy( - name="synergy-suffix-chu", - filter_left=filter_is_noun, - filter_right=filter_in_seq_set(1620400, 2083570), # 中 - description="suffix-chu", - score=12, - connector="-", - ) - - # noun + たち - # Use dynamic score based on noun length - longer nouns get higher synergy - # This helps 村人+たち (258+15=273) beat 村+人たち (263) - def_generic_synergy( - name="synergy-suffix-tachi", - filter_left=filter_is_noun, - filter_right=filter_in_seq_set(1416220), # たち - description="suffix-tachi", - score=lambda l, r: 10 + 5 * (l.end - l.start), # +5 per character - connector="-", - ) - - # noun + ぶり - def_generic_synergy( - name="synergy-suffix-buri", - filter_left=filter_is_noun, - filter_right=filter_in_seq_set(1361140), # ぶり - description="suffix-buri", - score=40, - connector="", - ) - - # noun + 性 - def_generic_synergy( - name="synergy-suffix-sei", - filter_left=filter_is_noun, - filter_right=filter_in_seq_set(1375260), # 性 - description="suffix-sei", - score=12, - connector="", - ) - - # お + noun (excluding cases where it splits a valid compound like ごみ) - # Create a combined filter that checks pos AND excludes specific seqs - def filter_o_noun_right(segment: Any) -> bool: - """Filter for nouns that can follow お/ご prefix.""" - # Must be a noun - if not filter_is_pos("n")(segment): - return False - # Exclude みの (straw raincoat) to prevent splitting ごみ - info = getattr(segment, 'info', {}) - seq_set = info.get('seq_set', set()) - excluded_seqs = {1634010, 2845080} # みの seqs - if seq_set.intersection(excluded_seqs): - return False - return True - - def_generic_synergy( - name="synergy-o-prefix", - filter_left=filter_in_seq_set(1270190), # お - filter_right=filter_o_noun_right, - description="o+noun", - score=10, - connector="", - ) - - # 未/不 + noun - def_generic_synergy( - name="synergy-kanji-prefix", - filter_left=filter_in_seq_set(2242840, 1922780, 2423740), # 未, 不 - filter_right=filter_is_pos("n"), - description="kanji prefix+noun", - score=15, - connector="", - ) - - # しちゃ/しては + いけない - def_generic_synergy( - name="synergy-shicha-ikenai", - filter_left=filter_is_compound_end(2028920), # は - filter_right=filter_in_seq_set(1000730, 1612750, 1409110, 2829697, 1587610), - description="shicha ikenai", - score=50, - connector=" ", - ) - - # しか + negative - def synergy_shika_negative(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - start = seg_list_left.end - end = seg_list_right.start - - if start != end: - return [] - - # Filter left for しか - filter_shika = filter_in_seq_set(1005460) - left_segments = [s for s in seg_list_left.segments if filter_shika(s)] - - if not left_segments: - return [] - - # Filter right for negative conjugation - right_segments = [] - for seg in seg_list_right.segments: - info = getattr(seg, 'info', {}) - conj = info.get('conj', []) - for cdata in conj: - if hasattr(cdata, 'prop') and cdata.prop and getattr(cdata.prop, 'neg', False): - right_segments.append(seg) - break - - if not right_segments: - return [] - - syn = Synergy( - description="shika+neg", - connector=" ", - score=50, - start=start, - end=end, - ) - - from himotoki.lookup import SegmentList - new_left = SegmentList( - segments=left_segments, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_segments, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_shika_negative) - - # の + 通り - def_generic_synergy( - name="synergy-no-toori", - filter_left=filter_in_seq_set(1469800), # の - filter_right=filter_in_seq_set(1432920), # 通り - description="no toori", - score=50, - connector=" ", - ) - - # counter + おき - def_generic_synergy( - name="synergy-oki", - filter_left=filter_is_pos("ctr"), - filter_right=filter_in_seq_set(2854117, 2084550), # おき - description="counter+oki", - score=20, - connector="", - ) - - # かどうか + は (whether or not + topic marker) - # This prevents misparses like はま+だ instead of は+まだ - def_generic_synergy( - name="synergy-kadouka-wa", - filter_left=filter_in_seq_set(2087300), # かどうか - filter_right=filter_in_seq_set(2028920), # は - description="kadouka+wa", - score=30, - connector=" ", - ) - - # particle + common adverb synergy - # This boosts patterns like は+まだ, には+まだ, も+まだ etc. - # Common adverbs that follow particles: - # まだ (1527110): still, yet - # もう (1010180): already, soon - # まず (1623080): first of all - # やはり (2084660): as expected - # すでに (1303920): already - # ずっと (1008930): continuously - # もっと (1010210): more - # とても (1008550): very - # かなり (1004920): considerably - # なかなか (1530760): quite, rather - # 少し (1340610): a little - # ちょっと (1008680): a little - # 全然 (1391950): not at all - # 絶対 (1391700): absolutely - # 本当に (1583020): really - # 実は (1311820): actually - # 確かに (1208880): certainly - # 多分 (1397270): probably - # きっと (1399930): surely - # たぶん (1397270): probably - # 結局 (1252670): after all - # 一応 (2423580): tentatively - # とりあえず (1541310): for now - # 相変わらず (1273140): as usual - # いつも (1216780): always - # たまに (1623560): occasionally - # 時々 (1320780): sometimes - # よく (1544660): often - # あまり (1010990): not very - COMMON_ADVERB_SEQS = { - 1527110, # まだ - 1010180, # もう - 1623080, # まず - 2084660, # やはり - 1303920, # すでに (既に) - 1008930, # ずっと - 1010210, # もっと - 1008550, # とても - 1004920, # かなり - 1530760, # なかなか - 1340610, # 少し - 1008680, # ちょっと - 1391950, # 全然 - 1391700, # 絶対 - 1583020, # 本当に - 1311820, # 実は - 1208880, # 確かに - 1397270, # 多分/たぶん - 1399930, # きっと - 1252670, # 結局 - 2423580, # 一応 - 1541310, # とりあえず - 1273140, # 相変わらず - 1216780, # いつも - 1623560, # たまに - 1320780, # 時々 - 1544660, # よく - 1010990, # あまり - } - - # Particles that can precede common adverbs - PARTICLE_SEQS = { - 2028920, # は - 2028930, # が - 2028940, # も - 2028990, # に - 2028980, # で - 2215430, # には - 2028950, # とは - 1007340, # だけ - 1525680, # まで - 1002980, # から - } - - def_generic_synergy( - name="synergy-particle-adverb", - filter_left=filter_in_seq_set(*PARTICLE_SEQS), - filter_right=filter_in_seq_set(*COMMON_ADVERB_SEQS), - description="particle+adverb", - score=20, # Strong synergy to beat noun+copula patterns - connector=" ", - ) - - # === BUG FIX: volitional + とも (1k3) === - # Add synergy for volitional form + とも (seq=1632180) - # This pattern means "even if" (e.g., 反対しようとも = even if they oppose) - # We need to boost とも when it follows a volitional form - def synergy_volitional_tomo(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for volitional form + とも.""" - start = seg_list_left.end - end = seg_list_right.start - - # Must be adjacent - if start != end: - return [] - - # Check if left has volitional conjugation - left_volitional = [] - for seg in seg_list_left.segments: - info = getattr(seg, 'info', {}) - conj = info.get('conj', []) - for cdata in conj: - if hasattr(cdata, 'prop') and cdata.prop: - if getattr(cdata.prop, 'conj_type', None) == 9: # CONJ_VOLITIONAL = 9 - left_volitional.append(seg) - break - - if not left_volitional: - return [] - - # Check if right is とも (seq=1632180) - filter_tomo = filter_in_seq_set(1632180) - right_tomo = [s for s in seg_list_right.segments if filter_tomo(s)] - - if not right_tomo: - return [] - - syn = Synergy( - description="volitional+tomo", - connector=" ", - score=50, # Strong synergy to prefer compound とも - start=start, - end=end, - ) - - from himotoki.lookup import SegmentList - new_left = SegmentList( - segments=left_volitional, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_tomo, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_volitional_tomo) - - # === BUG FIX: verb + なんて (v3f) === - # Add synergy for verb + なんて (seq=1188370) - # This prevents 提出させられるな + ん + て (imperative negative split) - # なんて is an expression meaning "such things as" that follows verb plain form - def synergy_verb_nante(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for verb + なんて.""" - start = seg_list_left.end - end = seg_list_right.start - - # Must be adjacent - if start != end: - return [] - - # Check if left is verb-like (non-negative, non-imperative) - left_verbs = [] - for seg in seg_list_left.segments: - info = getattr(seg, 'info', {}) - posi = info.get('posi', []) - verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', 'v5t', 'v5u', 'vk', 'vs', 'vs-i'} - - if verb_pos.intersection(posi): - # Check it's not imperative negative - conj = info.get('conj', []) - is_imperative_neg = False - for cdata in conj: - if hasattr(cdata, 'prop') and cdata.prop: - if getattr(cdata.prop, 'conj_type', None) == 10: # CONJ_IMPERATIVE - if getattr(cdata.prop, 'neg', False): - is_imperative_neg = True - break - - if not is_imperative_neg: - left_verbs.append(seg) - - if not left_verbs: - return [] - - # Check if right is なんて (seq=1188370) - filter_nante = filter_in_seq_set(1188370) - right_nante = [s for s in seg_list_right.segments if filter_nante(s)] - - if not right_nante: - return [] - - syn = Synergy( - description="verb+nante", - connector=" ", - score=60, # Strong synergy to prefer なんて as expression - start=start, - end=end, - ) - - from himotoki.lookup import SegmentList - new_left = SegmentList( - segments=left_verbs, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_nante, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_verb_nante) - - # === BUG FIX: し (particle) + ただ (adverb) (f5d) === - # Boost し+ただ over した+だ pattern - # し (seq=2086640) is a particle meaning "and/besides" - # ただ (seq=1538900) is an adverb meaning "only/just" - # Pattern: ねーしただ = "isn't, and just..." not "isn't, did, is..." - SEQ_SHI_PARTICLE = 2086640 - SEQ_TADA_ADV = 1538900 - - def synergy_shi_tada(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for し (particle) + ただ (adverb).""" - start = seg_list_left.end - end = seg_list_right.start - - # Must be adjacent - if start != end: - return [] - - # Check if left is し particle (seq=2086640) - filter_shi = filter_in_seq_set(SEQ_SHI_PARTICLE) - left_shi = [s for s in seg_list_left.segments if filter_shi(s)] - - if not left_shi: - return [] - - # Check if right is ただ (seq=1538900) - filter_tada = filter_in_seq_set(SEQ_TADA_ADV) - right_tada = [s for s in seg_list_right.segments if filter_tada(s)] - - if not right_tada: - return [] - - syn = Synergy( - description="shi+tada", - connector=" ", - score=10, # Small boost to prefer し+ただ over した+だ - start=start, - end=end, - ) - - from himotoki.lookup import SegmentList - new_left = SegmentList( - segments=left_shi, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_tada, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_shi_tada) - - # === BUG FIX: verb + よ (particle) (h0z) === - # Boost verb + よ (particle) over adjective stem interpretation - # よ (seq=2029090) is a sentence-ending emphasis particle - # The adjective stem of いい -> よ (euphonic change) causes misidentification - # When よ follows a verb, it's almost always the particle - SEQ_YO_PARTICLE = 2029090 - - def synergy_verb_yo(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for verb + よ (particle).""" - start = seg_list_left.end - end = seg_list_right.start - - # Must be adjacent - if start != end: - return [] - - # Check if left is a verb - left_verbs = [] - verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', 'v5t', 'v5u', 'vk', 'vs', 'vs-i'} - for seg in seg_list_left.segments: - info = getattr(seg, 'info', {}) - posi = info.get('posi', []) - if verb_pos.intersection(posi): - left_verbs.append(seg) - - if not left_verbs: - return [] - - # Check if right is よ particle (seq=2029090) AND is not conjugated - right_yo = [] - for seg in seg_list_right.segments: - info = getattr(seg, 'info', {}) - seq_set = info.get('seq_set', set()) - if SEQ_YO_PARTICLE in seq_set: - # Check that this is NOT a conjugated form - conj = info.get('conj', []) - is_conjugated = False - for cdata in conj: - if hasattr(cdata, 'prop') and cdata.prop: - if getattr(cdata.prop, 'conj_type', None) is not None: - is_conjugated = True - break - if not is_conjugated: - right_yo.append(seg) - - if not right_yo: - return [] - - syn = Synergy( - description="verb+yo", - connector=" ", - score=30, # Strong boost to prefer particle よ after verbs - start=start, - end=end, - ) - - from himotoki.lookup import SegmentList - new_left = SegmentList( - segments=left_verbs, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_yo, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_verb_yo) - - # === BUG FIX: copula + よ (particle) (v42) === - # Boost copula + よ (particle) to prevent よそ mis-segmentation - # When です/だ is followed by よ particle, boost that interpretation - # This prevents "ですよその" from being parsed as "です+よそ+の" - SEQ_DESU = 1628500 # です - SEQ_DA = 2089020 # だ - SEQ_YOSO = 1612400 # よそ (elsewhere) - used for blocking - - def synergy_copula_yo(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for copula + よ (particle) to prevent よそ mis-segmentation.""" - start = seg_list_left.end - end = seg_list_right.start - - # Must be adjacent - if start != end: - return [] - - # Check if left is copula (です or だ) - filter_copula = filter_in_seq_set(SEQ_DESU, SEQ_DA) - left_copula = [s for s in seg_list_left.segments if filter_copula(s)] - if not left_copula: - return [] - - # Check if right is よ particle (seq=SEQ_YO_PARTICLE, not conjugated) - right_yo = [] - for seg in seg_list_right.segments: - info = getattr(seg, 'info', {}) - seq_set = info.get('seq_set', set()) - if SEQ_YO_PARTICLE in seq_set: - # Check that this is NOT a conjugated form - conj = info.get('conj', []) - is_conjugated = False - for cdata in conj: - if hasattr(cdata, 'prop') and cdata.prop: - if getattr(cdata.prop, 'conj_type', None) is not None: - is_conjugated = True - break - if not is_conjugated: - right_yo.append(seg) - - if not right_yo: - return [] - - syn = Synergy( - description="copula+yo", - connector=" ", - score=40, # Strong boost to prefer particle よ after copula - start=start, - end=end, - ) - - from himotoki.lookup import SegmentList - new_left = SegmentList( - segments=left_copula, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_yo, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_copula_yo) - - # 前(まえ) + に synergy - boost noun reading over prefix reading - # When 前 is followed by に particle, prefer まえ (noun) over ぜん (prefix) - def synergy_mae_ni(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for 前(まえ) + に to prefer noun reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for seq=SEQ_MAE_NOUN (まえ reading) - filter_left = filter_in_seq_set(SEQ_MAE_NOUN) - left_mae = [s for s in seg_list_left.segments if filter_left(s)] - if not left_mae: - return [] - - # Filter right for に particle - filter_right = filter_in_seq_set(SEQ_NI) - right_ni = [s for s in seg_list_right.segments if filter_right(s)] - if not right_ni: - return [] - - # Create synergy with high score to push まえ over ぜん - syn = Synergy( - description="mae+ni", - connector=" ", - score=25, # Higher score to ensure まえ wins over ぜん - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_mae, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_ni, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_mae_ni) - - # の + 方(ほう) synergy - boost direction reading over person reading - # When の is followed by 方, prefer ほう (direction) over かた (person) - # Pattern: 〜の方 (towards/that direction) uses ほう reading - def synergy_no_hou(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for の + 方(ほう) to prefer direction reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for の particle - filter_left = filter_in_seq_set(SEQ_NO) - left_no = [s for s in seg_list_left.segments if filter_left(s)] - if not left_no: - return [] - - # Filter right for seq=SEQ_HOU_NOUN (ほう reading) - filter_right = filter_in_seq_set(SEQ_HOU_NOUN) - right_hou = [s for s in seg_list_right.segments if filter_right(s)] - if not right_hou: - return [] - - # Create synergy with high score to push ほう over かた - syn = Synergy( - description="no+hou", - connector=" ", - score=25, # Higher score to ensure ほう wins - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_no, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_hou, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_no_hou) - - # noun + 面(めん) synergy - boost aspect reading over face reading - # When a noun is followed by 面, prefer めん (aspect) over つら (face) - # Pattern: X面 (X aspect/side) like コスト面, 経済面 uses めん reading - def synergy_noun_men(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for noun + 面(めん) to prefer aspect reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for nouns - left_nouns = [s for s in seg_list_left.segments if filter_is_noun(s)] - if not left_nouns: - return [] - - # Filter right for seq=SEQ_MEN_NOUN (めん reading) - filter_right = filter_in_seq_set(SEQ_MEN_NOUN) - right_men = [s for s in seg_list_right.segments if filter_right(s)] - if not right_men: - return [] - - # Create synergy with high score to push めん over つら - syn = Synergy( - description="noun+men", - connector=" ", - score=25, # Higher score to ensure めん wins - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_nouns, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_men, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_noun_men) - - # verb/clause + 人(ひと) synergy - boost person reading over suffix reading - # When 人 follows a verb in plain form or a clause-ending word, - # it should be read as ひと (person), not じん (suffix). - # Pattern: 持つ人, 知らない人, いる人 -- all use ひと - # じん is only used as a suffix after nouns (日本人, アメリカ人) - def synergy_verb_hito(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for verb + 人(ひと) to prefer person reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for verbs or adjectives (clause-ending words) - left_verbs = [] - verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', - 'v5t', 'v5u', 'v5u-s', 'v5k-s', 'v5r-i', 'vk', 'vs', - 'vs-i', 'adj-i', 'adj-ix'} - for seg in seg_list_left.segments: - info = getattr(seg, 'info', {}) - posi = info.get('posi', []) - if verb_pos.intersection(posi): - left_verbs.append(seg) - - if not left_verbs: - return [] - - # Filter right for seq=SEQ_HITO_NOUN (ひと reading) - filter_right = filter_in_seq_set(SEQ_HITO_NOUN) - right_hito = [s for s in seg_list_right.segments if filter_right(s)] - if not right_hito: - return [] - - syn = Synergy( - description="verb+hito", - connector=" ", - score=25, - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_verbs, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_hito, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_verb_hito) - - # 人(ひと) + の synergy - boost person reading when followed by possessive - # When 人 is followed by の (possessive), it should be read as ひと (person). - # Pattern: 人の一生, 人の話, 人の気持ち -- all use ひと - # This complements the verb+hito synergy by handling sentence-initial 人 - SEQ_NO_PARTICLE = 1469800 # の (particle) - - def synergy_hito_no(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for 人(ひと) + の to prefer person reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for seq=SEQ_HITO_NOUN (ひと reading) - filter_left = filter_in_seq_set(SEQ_HITO_NOUN) - left_hito = [s for s in seg_list_left.segments if filter_left(s)] - if not left_hito: - return [] - - # Filter right for の (possessive particle) - filter_right = filter_in_seq_set(SEQ_NO_PARTICLE) - right_no = [s for s in seg_list_right.segments if filter_right(s)] - if not right_no: - return [] - - syn = Synergy( - description="hito+no", - connector=" ", - score=30, # Strong enough to override じん - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_hito, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_no, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_hito_no) - - # する + 中(なか) synergy - boost middle reading over suffix reading - # When 中 follows a verb (する中, 激化する中, etc.), it means "amid/while" - # and should be read as なか, not ちゅう (which is a suffix like 勉強中) - def synergy_verb_naka(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for verb + 中(なか) to prefer 'middle/amid' reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for verbs in plain form (non-past, non-negative) - left_verbs = [] - verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', - 'v5t', 'v5u', 'v5u-s', 'v5k-s', 'v5r-i', 'vk', 'vs', - 'vs-i'} - for seg in seg_list_left.segments: - info = getattr(seg, 'info', {}) - posi = info.get('posi', []) - if verb_pos.intersection(posi): - left_verbs.append(seg) - - if not left_verbs: - return [] - - # Filter right for seq=SEQ_NAKA_NOUN (なか reading) - filter_right = filter_in_seq_set(SEQ_NAKA_NOUN) - right_naka = [s for s in seg_list_right.segments if filter_right(s)] - if not right_naka: - return [] - - syn = Synergy( - description="verb+naka", - connector=" ", - score=25, - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_verbs, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_naka, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_verb_naka) - - # が + 止まる(とまる) synergy - prefer "to stop" over "to stay/remain" - # When 止まる follows が (subject marker), it usually means "stops" (とまる). - # "とどまる" (to stay/remain) typically takes に (にとどまる). - # Pattern: 時間が止まった, 車が止まった, 心臓が止まった - def synergy_ga_tomaru(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for が + 止まる(とまる) to prefer 'to stop' reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for が particle - filter_ga = filter_in_seq_set(SEQ_GA) - left_ga = [s for s in seg_list_left.segments if filter_ga(s)] - if not left_ga: - return [] - - # Filter right for とまる (seq=1310620, to stop) - filter_tomaru = filter_in_seq_set(SEQ_TOMARU) - right_tomaru = [s for s in seg_list_right.segments if filter_tomaru(s)] - if not right_tomaru: - return [] - - syn = Synergy( - description="ga+tomaru", - connector=" ", - score=25, - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_ga, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_tomaru, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_ga_tomaru) - - # は + 辛い(つらい) synergy - prefer "painful/hard" over "spicy" - # When 辛い follows は (topic marker), it usually means "hard/painful" (つらい). - # "からい" (spicy/hot) is primarily used in food contexts with direct modification. - # Pattern: リハビリは辛い, 別れは辛い, 現実は辛い - def synergy_wa_tsurai(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """Synergy for は + 辛い(つらい) to prefer 'painful/hard' reading.""" - from himotoki.lookup import SegmentList - - # Check serial - must be adjacent - if seg_list_left.end != seg_list_right.start: - return [] - - # Filter left for は particle - filter_wa = filter_in_seq_set(SEQ_WA) - left_wa = [s for s in seg_list_left.segments if filter_wa(s)] - if not left_wa: - return [] - - # Filter right for つらい (seq=1365860, painful/hard) - filter_tsurai = filter_in_seq_set(SEQ_TSURAI) - right_tsurai = [s for s in seg_list_right.segments if filter_tsurai(s)] - if not right_tsurai: - return [] - - syn = Synergy( - description="wa+tsurai", - connector=" ", - score=50, - start=seg_list_left.end, - end=seg_list_right.start, - ) - - new_left = SegmentList( - segments=left_wa, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - new_right = SegmentList( - segments=right_tsurai, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - - return [(new_right, syn, new_left)] - - register_synergy(synergy_wa_tsurai) - - - -# Initialize synergies on module load -_init_synergies() - - -def get_synergies(seg_list_left: Any, seg_list_right: Any) -> List[Tuple]: - """ - Get all synergies between two segment lists. - - Returns list of (new_right, synergy, new_left) tuples. - """ - results = [] - for fn in _synergy_list: - results.extend(fn(seg_list_left, seg_list_right)) - return results - - -# ============================================================================ -# Penalty List and Registration -# ============================================================================ - -_penalty_list: List[Callable] = [] - - -def register_penalty(func: Callable): - """Register a penalty function.""" - _penalty_list.append(func) - return func - - -def def_generic_penalty( - name: str, - test_left: Callable, - test_right: Callable, - description: str, - score: int, - serial: bool = True, - connector: str = " ", -): - """ - Define a generic penalty between two segment lists. - """ - def penalty_fn(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: - start = seg_list_left.end - end = seg_list_right.start - - # Check serial requirement - if serial and start != end: - return None - - if not test_left(seg_list_left): - return None - if not test_right(seg_list_right): - return None - - return Synergy( - description=description, - connector=connector, - score=score, - start=start, - end=end, - ) - - register_penalty(penalty_fn) - return penalty_fn - - -# Define penalties -def _init_penalties(): - """Initialize all penalty definitions.""" - - # と + は should be penalized to prefer compound とは (seq=2028950) - # The noun+prt synergy gives +14 to と+は which makes it beat とは (24 vs 22) - # This penalty counteracts that so とは wins - # NOTE: This must come BEFORE other penalties so it applies first - def has_seq_simple(seqs: set): - """Check if segment_list has any of the given seq numbers.""" - def _filter(segment_list: Any) -> bool: - segments = getattr(segment_list, 'segments', []) - for seg in segments: - info = getattr(seg, 'info', {}) - seq_set = info.get('seq_set', set()) - if seqs.intersection(seq_set): - return True - return False - return _filter - - def_generic_penalty( - name="penalty-to-wa", - test_left=has_seq_simple({1008490}), # と - test_right=has_seq_simple({2028920}), # は - description="to+wa-penalty", - score=-20, - serial=True, - ) - - # に + つれ should be penalized to prefer compound につれ (seq=2136050) - # につれ has score 36, but に(11)+つれ(40)=51 wins otherwise - # Need penalty of at least -(51-36)=-15, but make it larger to be safe - # つれ seqs include: 10351890, 1434020, 10097136, 1559290, 1434120, 10351981 - def_generic_penalty( - name="penalty-ni-tsure", - test_left=has_seq_simple({2028990}), # に - test_right=has_seq_simple({10351890, 1434020, 10097136, 1559290, 1434120, 10351981}), # つれ - description="ni+tsure-penalty", - score=-30, - serial=True, - ) - - # お + すすめ should be penalized to prefer compound おすすめ (seq=1002150) - # おすすめ has score 64, but お(10)+すすめ(90)=100 wins otherwise - # Need penalty of at least -36 to make them equal - # お seqs: 1343610 (common=12), 1485770 (common=5), 2089690, 2268350, 2603520, 2742870, 2826528 - # すすめ seqs: 1210900 (noun), 10074000 (verb conjugation), 1365980, 1365990 - def_generic_penalty( - name="penalty-o-susume", - test_left=has_seq_simple({1343610, 1485770, 2089690, 2268350, 2603520, 2742870, 2826528}), # お - test_right=has_seq_simple({1210900, 10074000, 1365980, 1365990}), # すすめ - description="o+susume-penalty", - score=-40, - serial=True, - ) - - # 人がい + たら should be penalized to prefer 人 + が + いたら - # 人がい is a conjugation of 人がいい (seq=10043332, 2250200) - # 人がい(120)+たら(68)=188 beats 人(16)+が(11)+いたら(90)=117 - # Need penalty of at least -71 to make them equal - # たら seqs: 1408160, 1416790, 2029050, 11435516, 11679733 - def_generic_penalty( - name="penalty-hitogai-tara", - test_left=has_seq_simple({10043332, 2250200}), # 人がい / 人がいい - test_right=has_seq_simple({1408160, 1416790, 2029050, 11435516, 11679733}), # たら - description="hitogai+tara-penalty", - score=-75, - serial=True, - ) - - # ご + みの should be penalized to prefer compound ごみ (seq=1369900) - # ごみ + の = 12 + 16 = 28 - # ご(10) + みの(12) = 22, plus o+noun synergy(+10) = 32 - # Need penalty of at least -6 to make ごみ + の win (28 > 26) - # Using -15 to ensure ごみ is strongly preferred - # ご (honorific prefix): seq=1270190 - # みの: seq=1634010 (straw raincoat), 2845080 (Mino province) - def_generic_penalty( - name="penalty-go-mino", - test_left=has_seq_simple({1270190}), # ご (honorific prefix) - test_right=has_seq_simple({1634010, 2845080}), # みの - description="go+mino-penalty", - score=-15, - serial=True, - ) - - # わかん + ない should be penalized to prefer compound わかんない (seq=2158960) - # わかんない has score 125, but わかん(110)+ない(40)=150 wins otherwise - # Need penalty of at least -(150-125)=-25 to make わかんない win - # Using -30 to ensure it's strongly preferred - # わかん seqs: 10256789, 10256836 (conjugated forms of 分かる) - # ない seqs: 10452328, 1529520 (negative adjective/suffix) - def_generic_penalty( - name="penalty-wakan-nai", - test_left=has_seq_simple({10256789, 10256836}), # わかん - test_right=has_seq_simple({10452328, 1529520}), # ない - description="wakan+nai-penalty", - score=-30, - serial=True, - ) - - # 知らん + けど should be penalized to prefer compound 知らんけど (seq=2856919) - # 知らんけど exists as a single expression (Kansai dialect "I dunno but") - # 知らん seqs include various conjugated forms - # けど seqs: 1004200 - def_generic_penalty( - name="penalty-shiran-kedo", - test_left=has_seq_simple({10350776}), # 知らん (conjugated 知る) - test_right=has_seq_simple({1004200}), # けど - description="shiran+kedo-penalty", - score=-100, # Strong penalty since 知らんけど is a specific expression - serial=True, - ) - - # はま + だ should be penalized to prefer は + まだ - # はま (seq=1490710, beach) + だ (copula) incorrectly beats は + まだ (still) - # はま+だ gets 66 (with noun+だ synergy), は+まだ gets 51 - # With particle+adverb synergy (+20), は+まだ gets 71 - # But we still need a penalty to be safe: -20 makes はま+だ = 46, safely below 71 - def_generic_penalty( - name="penalty-hama-da", - test_left=has_seq_simple({1490710, 2084350}), # はま (beach, other) - test_right=has_seq_simple({2089020}), # だ (copula) - description="hama+da-penalty", - score=-20, - serial=True, - ) - - # 人たち after single-kanji word should be penalized when compound word exists - # 村 + 人たち (263) vs 村人 + たち (258) - # This penalty makes 村+人たち = 263-10 = 253, so 村人+たち (258) wins - def penalty_hitotachi_split(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: - """Penalize single-kanji + 人たち to prefer compound + たち.""" - start = seg_list_left.end - end = seg_list_right.start - - # Must be serial - if start != end: - return None - - # Check if right is 人たち (seq=1368740) - segments_right = getattr(seg_list_right, 'segments', []) - has_hitotachi = False - for seg in segments_right: - info = getattr(seg, 'info', {}) - seq_set = info.get('seq_set', set()) - if 1368740 in seq_set: - has_hitotachi = True - break - - if not has_hitotachi: - return None - - # Check if left is a single-character word (likely kanji that could combine) - segments_left = getattr(seg_list_left, 'segments', []) - for seg in segments_left: - word = getattr(seg, 'word', None) - if word and hasattr(word, 'text'): - text = word.text - if len(text) == 1: - # Single character - check if it's kanji - from himotoki.characters import is_kanji - if is_kanji(text): - return Synergy( - description="single-kanji+hitotachi-penalty", - connector=" ", - score=-15, # Penalty to prefer compound+たち - start=start, - end=end, - ) - - return None - - register_penalty(penalty_hitotachi_split) - - # === BUG FIX: につけ (7g2) === - # に + つけ should be penalized to prefer compound につけ (seq=2840365) - # につけ is a grammatical expression meaning "whenever" - # つけ seqs: 1495750 (noun), conjugations of つける (10092135, 10092153, etc.) - def_generic_penalty( - name="penalty-ni-tsuke", - test_left=has_seq_simple({2028990}), # に - test_right=has_seq_simple({1495750, 10092135, 10092153, 10092153, 1495740}), # つけ - description="ni+tsuke-penalty", - score=-30, - serial=True, - ) - - # === BUG FIX: 未だに (bzl) === - # 未だ + に should be penalized to prefer compound 未だに (seq=1527140) - # 未だに is an adverb meaning "still" or "even now" - # 未だ seqs: 1527110 - def_generic_penalty( - name="penalty-mada-ni", - test_left=has_seq_simple({1527110}), # 未だ - test_right=has_seq_simple({2028990}), # に - description="mada+ni-penalty", - score=-30, - serial=True, - ) - - # === BUG FIX: と + も after volitional (1k3) === - # と + も should be penalized to prefer compound とも (seq=1632180) - # when following a volitional form - # とも is a particle meaning "even if" after volitional form - def_generic_penalty( - name="penalty-to-mo", - test_left=has_seq_simple({1008490}), # と - test_right=has_seq_simple({2028940}), # も - description="to+mo-penalty", - score=-30, - serial=True, - ) - - # === BUG FIX: たん boundary (klu) === - # verb stem + たん should be penalized when past+ん is intended - # たん (seq=2646370) is a prefix/noun, not past+explanatory - # This prevents 仕掛け + たん instead of 仕掛けた + ん - # We need to check if left is a verb stem (continuative form) - # and right is たん (which should be た + ん instead) - def penalty_verb_tan(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: - """Penalize verb stem + たん when past+ん is intended.""" - start = seg_list_left.end - end = seg_list_right.start - - # Must be serial - if start != end: - return None - - # Check if right is たん (seq=2646370) - segments_right = getattr(seg_list_right, 'segments', []) - has_tan = False - for seg in segments_right: - info = getattr(seg, 'info', {}) - seq_set = info.get('seq_set', set()) - if 2646370 in seq_set: - has_tan = True - break - - if not has_tan: - return None - - # Check if left ends with a word that could be a verb stem (masu-stem) - # Words like 仕掛け (noun) can also be verb stems - # Penalty should be applied regardless to prefer past+explanatory - segments_left = getattr(seg_list_left, 'segments', []) - for seg in segments_left: - info = getattr(seg, 'info', {}) - posi = info.get('posi', []) - # Check if it's a noun that could be a verb stem (many verb stems are also nouns) - # or a verb form - verb_pos = {'v1', 'v5r', 'v5s', 'v5k', 'v5g', 'v5b', 'v5m', 'v5n', 'v5t', 'v5u', 'vk', 'vs', 'vs-i', 'n'} - if verb_pos.intersection(posi): - return Synergy( - description="verb-stem+tan-penalty", - connector=" ", - score=-400, # Very strong penalty - diff is often 300+ points - start=start, - end=end, - ) - - return None - - register_penalty(penalty_verb_tan) - - # Short kana words together - def_generic_penalty( - name="penalty-short", - test_left=filter_short_kana(1), - test_right=filter_short_kana(1, except_list=['と']), - description="short", - score=-9, - serial=False, - ) - - # Semi-final particle not at end - def penalty_semi_final(seg_list_left: Any, seg_list_right: Any) -> Optional[Synergy]: - from himotoki.lookup import SEMI_FINAL_PRT - - # Check if left has semi-final particle - filter_fn = filter_in_seq_set(*SEMI_FINAL_PRT) - has_semi_final = any(filter_fn(s) for s in seg_list_left.segments) - - if not has_semi_final: - return None - - return Synergy( - description="semi-final not final", - connector=" ", - score=-15, - start=seg_list_left.end, - end=seg_list_right.start, - ) - - register_penalty(penalty_semi_final) - - -_init_penalties() - - -def get_penalties(seg_list_left: Any, seg_list_right: Any) -> List[Any]: - """ - Get penalties between two segment lists. - - Returns [seg_right, penalty, seg_left] if penalty applies, - otherwise [seg_right, seg_left]. - """ - for fn in _penalty_list: - penalty = fn(seg_list_left, seg_list_right) - if penalty: - return [seg_list_right, penalty, seg_list_left] - - return [seg_list_right, seg_list_left] - - -# ============================================================================ -# Segfilter List and Registration -# ============================================================================ - -_segfilter_list: List[Callable] = [] - - -def register_segfilter(func: Callable): - """Register a segfilter function.""" - _segfilter_list.append(func) - return func - - -def def_segfilter_must_follow( - name: str, - filter_left: Callable, - filter_right: Callable, - allow_first: bool = False, -): - """ - Define a segfilter where filter_right MUST follow filter_left. - - If filter_right matches but filter_left doesn't (and we're not at the start), - the matching segments are removed. - """ - def classify(filter_fn: Callable, items: List) -> Tuple[List, List]: - satisfies = [] - contradicts = [] - for item in items: - if filter_fn(item): - satisfies.append(item) - else: - contradicts.append(item) - return satisfies, contradicts - - def segfilter_fn(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - from himotoki.lookup import SegmentList - - satisfies_right, contradicts_right = classify(filter_right, seg_list_right.segments) - - # If nothing satisfies filter_right, pass through - if not satisfies_right: - return [(seg_list_left, seg_list_right)] - - # If first position and allowed, pass through - if allow_first and seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # If not adjacent, only allow contradicts - if seg_list_left is None or seg_list_left.end != seg_list_right.start: - if contradicts_right: - new_right = SegmentList( - segments=contradicts_right, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - return [(seg_list_left, new_right)] - return [] - - # Check left side - satisfies_left, contradicts_left = classify(filter_left, seg_list_left.segments) - - results = [] - - # If left has contradicts, allow those with right contradicts - if contradicts_left and contradicts_right: - results.append(( - seg_list_left, - SegmentList( - segments=contradicts_right, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )) - - # If left satisfies, allow with right satisfies - if satisfies_left: - results.append(( - SegmentList( - segments=satisfies_left, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ), - SegmentList( - segments=satisfies_right, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )) - - if not results and contradicts_right: - results.append(( - seg_list_left, - SegmentList( - segments=contradicts_right, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )) - - return results if results else [(seg_list_left, seg_list_right)] - - register_segfilter(segfilter_fn) - return segfilter_fn - - -# Define segfilters -def _init_segfilters(): - """Initialize all segfilter definitions.""" - - # Auxiliary verbs must follow continuative form - AUX_VERBS = {1342560} # 初める/そめる - def_segfilter_must_follow( - name="segfilter-aux-verb", - filter_left=filter_is_conjugation(13), # Continuative - filter_right=filter_in_seq_set(*AUX_VERBS), - ) - - # いる must not follow 終わる (つ + いる conflict) - def_segfilter_must_follow( - name="segfilter-tsu-iru", - filter_left=lambda s: not filter_in_seq_set(2221640)(s), - filter_right=filter_in_seq_set(1577980), # いる - allow_first=True, - ) - - # ん/んだ must not follow simple particles - def_segfilter_must_follow( - name="segfilter-n", - filter_left=lambda s: not filter_in_seq_set_simple(*NOUN_PARTICLES)(s), - filter_right=filter_in_seq_set(2139720, 2849370, 2849387), # ん, んだ - allow_first=True, - ) - - # を + 枯らす - def_segfilter_must_follow( - name="segfilter-wokarasu", - filter_left=filter_in_seq_set(2029010), # を - filter_right=filter_in_seq_set(2087020), - ) - - # Bad endings - def_segfilter_must_follow( - name="segfilter-badend", - filter_left=lambda s: False, - filter_right=filter_is_compound_end_text("ちゃい", "いか", "とか", "とき", "い"), - ) - - # じゃない must not follow は compound - def_segfilter_must_follow( - name="segfilter-janai", - filter_left=lambda s: not filter_is_compound_end(2028920)(s), - filter_right=filter_in_seq_set(1529520, 1296400, 2139720), # ない, ある, ん - allow_first=True, - ) - - # だ + する (dashi problem) - def segfilter_dashi_fn(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - from himotoki.lookup import SegmentList - - # Right must be する/して/し - filter_right = filter_in_seq_set(1157170, 2424740, 1305070) - satisfies_right = [s for s in seg_list_right.segments if filter_right(s)] - contradicts_right = [s for s in seg_list_right.segments if not filter_right(s)] - - if not satisfies_right: - return [(seg_list_left, seg_list_right)] - - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Left must not be だ without で - def left_ok(s): - info = getattr(s, 'info', {}) - seq_set = info.get('seq_set', []) - if 2089020 not in seq_set: # だ - return True - if 2028980 in seq_set: # で - return True - return False - - satisfies_left = [s for s in seg_list_left.segments if left_ok(s)] - - if satisfies_left: - return [(seg_list_left, seg_list_right)] - - if contradicts_right: - return [( - seg_list_left, - SegmentList( - segments=contradicts_right, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )] - - return [] - - register_segfilter(segfilter_dashi_fn) - - # Honorifics must follow noun-like words - HONORIFICS = {1247260} # 君 - def_segfilter_must_follow( - name="segfilter-honorific", - filter_left=lambda s: not filter_in_seq_set(*NOUN_PARTICLES)(s), - filter_right=filter_in_seq_set(*HONORIFICS), - ) - - # 君/くん (suffix) must NOT be followed by particles - # When 君 is followed by a particle (と, は, が, etc.), it's the pronoun きみ, - # not the suffix くん. Remove くん from candidates when followed by particles. - from himotoki.constants import SEQ_KUN, NOUN_PARTICLES as NP - def segfilter_kun_before_particle(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Remove くん (suffix) when followed by a particle.""" - from himotoki.lookup import SegmentList - - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - # Check if right is a particle - filter_particle = filter_in_seq_set(*NP) - right_is_particle = any(filter_particle(s) for s in seg_list_right.segments) - - if not right_is_particle: - return [(seg_list_left, seg_list_right)] - - # Right is a particle - remove くん from left - filter_kun = filter_in_seq_set(SEQ_KUN) - left_without_kun = [s for s in seg_list_left.segments if not filter_kun(s)] - - if not left_without_kun: - # All segments were くん - this path is blocked - return [] - - new_left = SegmentList( - segments=left_without_kun, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ) - return [(new_left, seg_list_right)] - - register_segfilter(segfilter_kun_before_particle) - - # === BUG FIX: Block patterns where compound expressions exist === - - # Segfilter: Block に + つけ to prefer につけ (7g2) - # につけ (seq=2840365) is a grammatical expression "whenever" - def segfilter_ni_tsuke(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Block に + つけ pattern to prefer につけ as compound.""" - from himotoki.lookup import SegmentList - - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - # Check if left is に (seq=2028990) - filter_ni = filter_in_seq_set(2028990) - left_ni = [s for s in seg_list_left.segments if filter_ni(s)] - left_other = [s for s in seg_list_left.segments if not filter_ni(s)] - - # Check if right is つけ (seq=1495750 or conjugations) - filter_tsuke = filter_in_seq_set(1495750, 10092135, 10092153, 1495740) - right_tsuke = [s for s in seg_list_right.segments if filter_tsuke(s)] - right_other = [s for s in seg_list_right.segments if not filter_tsuke(s)] - - # If not the pattern we're blocking, pass through - if not left_ni or not right_tsuke: - return [(seg_list_left, seg_list_right)] - - results = [] - # Allow left_other with any right - if left_other: - results.append(( - SegmentList( - segments=left_other, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ), - seg_list_right, - )) - - # Allow left_ni with right_other (non-tsuke) - if right_other: - results.append(( - seg_list_left, - SegmentList( - segments=right_other, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )) - - return results if results else [] - - register_segfilter(segfilter_ni_tsuke) - - # Segfilter: Block 未だ + に to prefer 未だに (bzl) - # 未だに (seq=1527140) is an adverb "still/even now" - def segfilter_mada_ni(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Block 未だ + に pattern to prefer 未だに as compound.""" - from himotoki.lookup import SegmentList - - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - # Check if left is 未だ (seq=1527110) - filter_mada = filter_in_seq_set(1527110) - left_mada = [s for s in seg_list_left.segments if filter_mada(s)] - left_other = [s for s in seg_list_left.segments if not filter_mada(s)] - - # Check if right is に (seq=2028990) - filter_ni = filter_in_seq_set(2028990) - right_ni = [s for s in seg_list_right.segments if filter_ni(s)] - right_other = [s for s in seg_list_right.segments if not filter_ni(s)] - - # If not the pattern we're blocking, pass through - if not left_mada or not right_ni: - return [(seg_list_left, seg_list_right)] - - results = [] - # Allow left_other with any right - if left_other: - results.append(( - SegmentList( - segments=left_other, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ), - seg_list_right, - )) - - # Allow left_mada with right_other (non-ni) - if right_other: - results.append(( - seg_list_left, - SegmentList( - segments=right_other, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )) - - return results if results else [] - - register_segfilter(segfilter_mada_ni) - - # Segfilter: Block と + も to prefer とも (1k3) - # とも (seq=1632180) is a particle meaning "even if" - def segfilter_to_mo(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Block と + も pattern to prefer とも as compound particle.""" - from himotoki.lookup import SegmentList - - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - # Check if left is と (seq=1008490) - filter_to = filter_in_seq_set(1008490) - left_to = [s for s in seg_list_left.segments if filter_to(s)] - left_other = [s for s in seg_list_left.segments if not filter_to(s)] - - # Check if right is も (seq=2028940) - filter_mo = filter_in_seq_set(2028940) - right_mo = [s for s in seg_list_right.segments if filter_mo(s)] - right_other = [s for s in seg_list_right.segments if not filter_mo(s)] - - # If not the pattern we're blocking, pass through - if not left_to or not right_mo: - return [(seg_list_left, seg_list_right)] - - results = [] - # Allow left_other with any right - if left_other: - results.append(( - SegmentList( - segments=left_other, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ), - seg_list_right, - )) - - # Allow left_to with right_other (non-mo) - if right_other: - results.append(( - seg_list_left, - SegmentList( - segments=right_other, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )) - - return results if results else [] - - register_segfilter(segfilter_to_mo) - - # === BUG FIX: Block verb-negative-imperative + ん (nwd, 5zp) === - # Pattern: verb ending with negative imperative な + ん should be blocked - # This forces the segmenter to use verb + なんて instead of verbな + ん + て - # Example: 発動させるな + ん + て → 発動させる + なんて - SEQ_N_PARTICLE = 2139720 # ん (particle/interjection) - - def segfilter_neg_imperative_n(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Block verb-negative-imperative + ん pattern to prefer verb + なんて.""" - from himotoki.lookup import SegmentList - - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - # Check if left has any negative imperative verbs - left_neg_imp = [] - left_other = [] - for seg in seg_list_left.segments: - info = getattr(seg, 'info', {}) - conj = info.get('conj', []) - is_neg_imperative = False - for cdata in conj: - if hasattr(cdata, 'prop') and cdata.prop: - # Check for imperative + negative - if getattr(cdata.prop, 'conj_type', None) == 10: # CONJ_IMPERATIVE - if getattr(cdata.prop, 'neg', False): - is_neg_imperative = True - break - - if is_neg_imperative: - left_neg_imp.append(seg) - else: - left_other.append(seg) - - # Check if right is ん (seq=2139720) - filter_n = filter_in_seq_set(SEQ_N_PARTICLE) - right_n = [s for s in seg_list_right.segments if filter_n(s)] - right_other = [s for s in seg_list_right.segments if not filter_n(s)] - - # If not the pattern we're blocking, pass through - if not left_neg_imp or not right_n: - return [(seg_list_left, seg_list_right)] - - results = [] - # Allow left_other with any right - if left_other: - results.append(( - SegmentList( - segments=left_other, - start=seg_list_left.start, - end=seg_list_left.end, - matches=seg_list_left.matches, - ), - seg_list_right, - )) - - # Allow left_neg_imp with right_other (non-ん) - if right_other: - results.append(( - seg_list_left, - SegmentList( - segments=right_other, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )) - - return results if results else [] - - register_segfilter(segfilter_neg_imperative_n) - - # === Segfilter: Remove いくさ reading after nouns === - # 戦(いくさ, seq=1587140) is an archaic standalone word for "war". - # When 戦 follows a noun, it should be read as せん (match/battle) - # as part of a compound (e.g., タイトル戦). Since せん gets culled - # by IDENTICAL_WORD_SCORE_CUTOFF (score 5 vs いくさ score 16), - # removing いくさ from the split path lets the compound path win. - from himotoki.constants import SEQ_IKUSA_NOUN - def segfilter_noun_ikusa(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Remove いくさ reading of 戦 when preceded by a noun.""" - from himotoki.lookup import SegmentList - - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - # Check if left has a noun - left_has_noun = any(filter_is_noun(s) for s in seg_list_left.segments) - if not left_has_noun: - return [(seg_list_left, seg_list_right)] - - # Remove いくさ from right segments - filter_ikusa = filter_in_seq_set(SEQ_IKUSA_NOUN) - right_without_ikusa = [s for s in seg_list_right.segments if not filter_ikusa(s)] - - if not right_without_ikusa: - # All segments were いくさ - block this split so compound wins - return [] - - if len(right_without_ikusa) == len(seg_list_right.segments): - # No いくさ found, pass through unchanged - return [(seg_list_left, seg_list_right)] - - new_right = SegmentList( - segments=right_without_ikusa, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ) - return [(seg_list_left, new_right)] - - register_segfilter(segfilter_noun_ikusa) - - # === Segfilter: Block ないよう (内容/内用/内洋) when matched from kana === - # When ないよう appears in hiragana, it's almost always ない+よう (negative purposive) - # rather than the kanji words 内容/内用/内洋. Only allow these seqs when matched from kanji. - SEQ_NAIYOU_KANA = {1459400, 1459440, 2862582} # All words with reading ないよう - def segfilter_naiyou_kana(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Filter out 内容/内用/内洋 when matched from kana (ないよう) instead of kanji.""" - from himotoki.lookup import SegmentList - - def is_naiyou_from_kana(seg): - if not hasattr(seg, 'word'): - return False - if seg.word.seq not in SEQ_NAIYOU_KANA: - return False - # Check if this came from kana matching - return seg.word.word_type == 'kana' - - filtered = [s for s in seg_list_right.segments if not is_naiyou_from_kana(s)] - - if len(filtered) == len(seg_list_right.segments): - # Nothing filtered, pass through - return [(seg_list_left, seg_list_right)] - - if not filtered: - # All filtered out - return empty result so this path is blocked - # The alternative path (ない + よう) should be taken instead - return [] - - return [( - seg_list_left, - SegmentList( - segments=filtered, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )] - - register_segfilter(segfilter_naiyou_kana) - - # === Segfilter: Block ところが (conjunction) after rentaikei forms === - # When ところが follows a noun-modifying verb form (rentaikei), it should be - # split as ところ (place) + が (particle), not the conjunction "however". - # Rentaikei forms: Non-past (type 1), Past (type 2), or any verb/adj that can modify nouns - # Example: 分からないところがある → 分からない + ところ + が + ある - SEQ_TOKOROGA = 1008570 # ところが (conjunction) - RENTAIKEI_CONJ_TYPES = {1, 2} # Non-past, Past - both can modify nouns - - def segfilter_tokoroga(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Filter out ところが (conjunction) when preceded by a noun-modifying form.""" - from himotoki.lookup import SegmentList - - # If no left context, allow ところが (sentence-initial is valid as conjunction) - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - def is_tokoroga_conj(seg): - return seg.word.seq == SEQ_TOKOROGA - - def is_rentaikei(seg): - """Check if segment ends with a noun-modifying (rentaikei) form.""" - conj_list = seg.info.get('conj', []) - if not conj_list: - # No conjugation - check if it's a noun or noun-like - posi = seg.info.get('posi', []) - if any(p in posi for p in ['n', 'adj-i', 'adj-na']): - return True - return False - # Check if any conjugation is rentaikei (can modify nouns) - for conj_data in conj_list: - if hasattr(conj_data, 'prop') and conj_data.prop: - if conj_data.prop.conj_type in RENTAIKEI_CONJ_TYPES: - return True - return False - - # Filter segments - tokoroga = [s for s in seg_list_right.segments if is_tokoroga_conj(s)] - other = [s for s in seg_list_right.segments if not is_tokoroga_conj(s)] - - # If no ところが, pass through - if not tokoroga: - return [(seg_list_left, seg_list_right)] - - # Check if left has rentaikei forms - rentaikei_left = [s for s in seg_list_left.segments if is_rentaikei(s)] - - # If left has rentaikei, block ところが (conjunction) - if rentaikei_left: - if other: - return [( - seg_list_left, - SegmentList( - segments=other, - start=seg_list_right.start, - end=seg_list_right.end, - matches=seg_list_right.matches, - ), - )] - # No other options - block this path entirely - return [] - - # No rentaikei on left, allow ところが - return [(seg_list_left, seg_list_right)] - - register_segfilter(segfilter_tokoroga) - - # === Segfilter: Block たん/たんだ after masu-stem verb forms === - # たん/たんだ as standalone words should not follow - # a masu-stem (continuative form) or noun that could be a verb stem. - # It should be た + ん (+ だ) instead. - # Example: 仕掛けたんだ → 仕掛けた + ん + だ, NOT 仕掛け + たん + だ - - def segfilter_tan_after_stem(seg_list_left: Optional[Any], seg_list_right: Any) -> List[Tuple]: - """Block たん/たんだ when preceded by continuative/masu-stem form.""" - from himotoki.lookup import SegmentList - - # If no left context, allow たん at sentence start - if seg_list_left is None: - return [(seg_list_left, seg_list_right)] - - # Must be adjacent - if seg_list_left.end != seg_list_right.start: - return [(seg_list_left, seg_list_right)] - - def is_tan_tanda(seg): - """Check if segment text is たん or たんだ.""" - return seg.word.text in ('たん', 'たんだ') - - def is_masu_stem_or_verb_noun(seg): - """Check if segment is a masu-stem form or a noun that could be a verb stem.""" - conj_list = seg.info.get('conj', []) - if conj_list: - for conj_data in conj_list: - if hasattr(conj_data, 'prop') and conj_data.prop: - # Conj type 13 = Continuative (~masu stem) - if conj_data.prop.conj_type == 13: - return True - # Also check if it's a noun that matches a verb's masu-stem pattern - # Many nouns like 仕掛け are also verb stems - posi = seg.info.get('posi', []) - if 'n' in posi: - # Check if the word looks like a potential verb stem - # (ends with け/き/し/り/ち/み/び/ぎ/え/い/etc.) - text = seg.word.text - if text and any(text.endswith(k) for k in 'けきしりちみびぎにえいてれねめべげ'): - return True - return False - - # Filter segments - tan_segs = [s for s in seg_list_right.segments if is_tan_tanda(s)] - other_segs = [s for s in seg_list_right.segments if not is_tan_tanda(s)] - - # If no たん/たんだ, pass through - if not tan_segs: - return [(seg_list_left, seg_list_right)] - - # Check if left has masu-stem or verb-like noun - has_stem = any(is_masu_stem_or_verb_noun(s) for s in seg_list_left.segments) - - if has_stem: - # Block たん/たんだ when preceded by masu-stem/verb-noun - if other_segs: - # Return only non-たん segments - new_right = SegmentList( - start=seg_list_right.start, - end=seg_list_right.end, - segments=other_segs - ) - return [(seg_list_left, new_right)] - else: - # No alternatives - block this path - return [] - - # Not a stem, allow たん/たんだ - return [(seg_list_left, seg_list_right)] - - register_segfilter(segfilter_tan_after_stem) - - -_init_segfilters() - - -def apply_segfilters(seg_left: Optional[Any], seg_right: Any) -> List[Tuple]: - """ - Apply all segfilters to a pair of segment lists. - - Returns list of (seg_left, seg_right) pairs that pass all filters. - """ - splits = [(seg_left, seg_right)] - - for segfilter in _segfilter_list: - new_splits = [] - for seg_l, seg_r in splits: - new_splits.extend(segfilter(seg_l, seg_r)) - splits = new_splits - - return splits +"""Backward-compatible shim for himotoki.synergies.""" +from himotoki.grammar.synergies import * # noqa: F401,F403 +from himotoki.grammar.synergy_filters import * # noqa: F401,F403 +from himotoki.constants import NOUN_PARTICLES # noqa: F401 diff --git a/himotoki/trie.py b/himotoki/trie.py index 1c8278f..4c968ab 100644 --- a/himotoki/trie.py +++ b/himotoki/trie.py @@ -9,6 +9,8 @@ before hitting the database - most substrings don't exist in the dictionary. """ +import os +from pathlib import Path from typing import Optional import marisa_trie @@ -17,6 +19,28 @@ _WORD_TRIE: Optional[marisa_trie.Trie] = None +def get_trie_path(db_path: Path) -> Path: + """Return path for persisted trie file next to the database.""" + db_path = Path(db_path) + if db_path.suffix == ".db": + return db_path.with_suffix(".trie") + return db_path.parent / f"{db_path.name}.trie" + + +def _get_db_path_from_session(session) -> Path: + """Resolve database path from session or environment.""" + env_path = os.environ.get("HIMOTOKI_DB_PATH") + if env_path: + return Path(env_path) + + bind = session.get_bind() + if bind is not None and bind.url.database: + return Path(bind.url.database) + + from himotoki.db.connection import _get_default_db_path + return _get_default_db_path() + + def get_word_trie() -> Optional[marisa_trie.Trie]: """Get the initialized trie, or None if not ready.""" return _WORD_TRIE @@ -27,11 +51,27 @@ def is_trie_ready() -> bool: return _WORD_TRIE is not None +def _build_trie_from_db(session) -> marisa_trie.Trie: + """Build trie from database surface forms.""" + conn = session.connection().connection + cursor = conn.cursor() + + cursor.execute( + "SELECT text FROM kana_text " + "UNION ALL " + "SELECT text FROM kanji_text" + ) + rows = cursor.fetchall() + + return marisa_trie.Trie(row[0] for row in rows) + + def init_word_trie(session) -> marisa_trie.Trie: """ Initialize the word trie from database. Loads all unique surface forms from kanji_text and kana_text tables. + Persists trie to disk next to the database for faster subsequent loads. Called during warm_up(). Args: @@ -43,22 +83,23 @@ def init_word_trie(session) -> marisa_trie.Trie: global _WORD_TRIE if _WORD_TRIE is not None: return _WORD_TRIE - - # Get raw connection for fast cursor iteration - conn = session.connection().connection - cursor = conn.cursor() - - # Use UNION ALL (faster than UNION - no dedup overhead) - # marisa-trie handles duplicates automatically - cursor.execute( - "SELECT text FROM kana_text " - "UNION ALL " - "SELECT text FROM kanji_text" - ) - rows = cursor.fetchall() - - # Build marisa-trie (handles duplicates, ~50-80MB for 9M entries) - _WORD_TRIE = marisa_trie.Trie(row[0] for row in rows) + + db_path = _get_db_path_from_session(session) + trie_path = get_trie_path(db_path) + + if db_path.exists() and trie_path.exists(): + trie_mtime = trie_path.stat().st_mtime + db_mtime = db_path.stat().st_mtime + if trie_mtime >= db_mtime: + _WORD_TRIE = marisa_trie.Trie().mmap(str(trie_path)) + return _WORD_TRIE + + _WORD_TRIE = _build_trie_from_db(session) + + if db_path.exists(): + trie_path.parent.mkdir(parents=True, exist_ok=True) + _WORD_TRIE.save(str(trie_path)) + return _WORD_TRIE @@ -94,8 +135,6 @@ def trie_has_prefix(prefix: str) -> bool: """ if _WORD_TRIE is None: return True # Fallback: assume it might exist - # marisa_trie.keys(prefix) returns all keys starting with prefix - # We just need to check if there's at least one try: next(iter(_WORD_TRIE.iterkeys(prefix))) return True diff --git a/himotoki/types.py b/himotoki/types.py new file mode 100644 index 0000000..e81991a --- /dev/null +++ b/himotoki/types.py @@ -0,0 +1,370 @@ +""" +Data types for word lookup and segmentation. +""" + +from dataclasses import dataclass, field +from typing import Optional, List, Dict, Tuple, Union, Any + +from sqlalchemy.orm import Session + +from himotoki.raw_types import RawKanaReading, RawKanjiReading +from himotoki.db.models import KanjiText, KanaText, ConjProp + + +def _is_counter_text(obj) -> bool: + """Check if object is a CounterText without importing the class.""" + return type(obj).__name__ == 'CounterText' + + +@dataclass(slots=True) +class ConjData: + """ + Conjugation data for a word match. + Tracks the conjugation chain from conjugated form to root. + """ + seq: int # Conjugated entry seq + from_seq: int # Root entry seq + via: Optional[int] # Intermediate seq for secondary conjugations + prop: Optional[ConjProp] # Conjugation property + src_map: List[Tuple[str, str]] = field(default_factory=list) # (conjugated_text, source_text) pairs + + +@dataclass +class WordMatch: + """ + Represents a word found in the database. + Wraps either a KanjiText/KanaText ORM object OR a lightweight RawKana/KanjiReading. + """ + reading: Union[KanjiText, KanaText, RawKanaReading, RawKanjiReading] + conjugations: Optional[List[int]] = None # List of conjugation IDs, or :root marker + hinted: bool = False + # Cached properties for performance (avoid repeated isinstance checks) + _word_type: Optional[str] = field(default=None, repr=False) + _seq: Optional[int] = field(default=None, repr=False) + _text: Optional[str] = field(default=None, repr=False) + + def __post_init__(self): + # Pre-compute cached properties on creation + # Detect type: ORM objects have __table__, raw namedtuples don't + # For raw types: RawKanaReading has 'best_kanji', RawKanjiReading has 'best_kana' + if isinstance(self.reading, (KanjiText, KanaText)): + # ORM object + self._word_type = 'kanji' if isinstance(self.reading, KanjiText) else 'kana' + elif isinstance(self.reading, RawKanaReading): + self._word_type = 'kana' + elif isinstance(self.reading, RawKanjiReading): + self._word_type = 'kanji' + else: + # Fallback: check for best_kanji attribute (kana has it, kanji doesn't) + self._word_type = 'kana' if hasattr(self.reading, 'best_kanji') else 'kanji' + self._seq = self.reading.seq + self._text = self.reading.text + + @property + def seq(self) -> int: + return self._seq + + @property + def text(self) -> str: + return self._text + + @property + def common(self) -> Optional[int]: + return self.reading.common + + @property + def ord(self) -> int: + return self.reading.ord + + @property + def word_type(self) -> str: + """Returns 'kanji' or 'kana' based on reading type.""" + return self._word_type + + @property + def is_root(self) -> bool: + """True if this is marked as a root form (not conjugated).""" + return self.conjugations == 'root' + + @property + def is_compound(self) -> bool: + """Always False for simple WordMatch.""" + return False + + @property + def components(self) -> List[str]: + """Return empty list for simple words (no components).""" + return [] + + def __repr__(self): + return f"" + + +@dataclass +class CompoundWord: + """ + Compound word made of 2 or more words joined together. + + Ports ichiran's compound-text class from dict.lisp lines 608-670. + Compound words are created when a primary word is joined with a suffix + (e.g., 食べている = 食べ + て + いる). + + Abbreviation compounds (is_abbrev=True) are scored differently - they use + the original word's mora length instead of the abbreviated compound length. + This matches Ichiran's proxy-text behavior for abbreviation suffixes. + """ + text: str # Full compound text + kana: str # Full kana reading + primary: WordMatch # Primary word (the main content word) + words: List[WordMatch] # All words in the compound + score_mod: Union[float, List[float]] = 0.0 # Score modifier(s) + score_base: Optional[WordMatch] = None # Base for scoring (usually primary) + is_abbrev: bool = False # True for abbreviation compounds (e.g., nai-x: ず/ざる/ぬ) + + @property + def seq(self) -> int: + """Returns primary seq for compound (not a list).""" + return self.primary.seq + + @property + def is_compound(self) -> bool: + """Always True for CompoundWord.""" + return True + + @property + def components(self) -> List[str]: + """Return component texts.""" + return [w.text for w in self.words] + + @property + def reading(self): + """Reading from primary word (for compatibility with calc_score).""" + return self.primary.reading + + @property + def is_root(self) -> bool: + """Compound words are never roots.""" + return False + + @property + def common(self) -> Optional[int]: + """Common rating from primary word.""" + return self.primary.common + + @property + def ord(self) -> int: + """Ord from primary word.""" + return self.primary.ord + + @property + def word_type(self) -> str: + """Word type from primary word.""" + return self.primary.word_type + + @property + def conjugations(self) -> Optional[List[int]]: + """Conjugations from last word in compound.""" + if self.words: + return self.words[-1].conjugations + return None + + @conjugations.setter + def conjugations(self, value): + """Set conjugations on last word.""" + if self.words: + self.words[-1].conjugations = value + + def get_score_base(self) -> WordMatch: + """Get the base word for scoring.""" + return self.score_base or self.primary + + def get_conjugation_info(self, session: 'Session') -> Dict[str, Any]: + """ + Get conjugation info from final word. + + Extracts conj_type, neg, fml, and source_text from the final word + in the compound. + + Args: + session: Database session for looking up conjugation data + + Returns: + Dict with keys: conj_type, neg, fml, source_text + """ + if not self.words: + return { + 'conj_type': None, + 'neg': False, + 'fml': False, + 'source_text': None, + } + + last_word = self.words[-1] + from himotoki.lookup.conj_data import get_word_conj_data + conj_data = get_word_conj_data(session, last_word) + + if not conj_data: + return { + 'conj_type': None, + 'neg': False, + 'fml': False, + 'source_text': None, + } + + # Get the first conjugation data entry + cd = conj_data[0] + prop = cd.prop + + # Get conj_type name + conj_type = None + neg = False + fml = False + if prop: + conj_type = prop.conj_type + neg = prop.neg if prop.neg else False + fml = prop.fml if prop.fml else False + + # Get source_text from src_map + source_text = None + if cd.src_map: + for text, src_text in cd.src_map: + if text == last_word.text: + source_text = src_text + break + + return { + 'conj_type': conj_type, + 'neg': neg, + 'fml': fml, + 'source_text': source_text, + } + + def __repr__(self): + return f"" + + +def adjoin_word( + word1: Union[WordMatch, 'CompoundWord'], + word2: WordMatch, + text: Optional[str] = None, + kana: Optional[str] = None, + score_mod: float = 0.0, + score_base: Optional[WordMatch] = None, + is_abbrev: bool = False, +) -> CompoundWord: + """ + Create compound word from 2 words. + + Ports ichiran's adjoin-word from dict.lisp lines 632-654. + + Args: + word1: Primary word or existing compound + word2: Word to append + text: Override text (default: concatenate both texts) + kana: Override kana (default: concatenate both kanas) + score_mod: Score modifier for this join + score_base: Base for scoring + is_abbrev: True if this is an abbreviation suffix (affects scoring) + + Returns: + CompoundWord combining both words + """ + # Default text and kana by concatenation + if text is None: + text = word1.text + word2.text + if kana is None: + # Derive kana from each word's kana/reading + def get_word_kana(w): + if isinstance(w, CompoundWord): + return w.kana + # For WordMatch, get kana from reading + if hasattr(w, 'reading'): + # KanjiText/RawKanjiReading has text=kanji, best_kana=kana reading + # KanaText/RawKanaReading has text=kana directly + if isinstance(w.reading, (KanjiText, RawKanjiReading)): + return w.reading.best_kana or w.reading.text + elif hasattr(w.reading, 'text'): + return w.reading.text + return w.text + kana = get_word_kana(word1) + get_word_kana(word2) + + if isinstance(word1, CompoundWord): + # Append to existing compound + word1.text = text + word1.kana = kana + word1.words = word1.words + [word2] + # Accumulate score_mod + if isinstance(word1.score_mod, list): + word1.score_mod = [score_mod] + word1.score_mod + else: + word1.score_mod = [score_mod, word1.score_mod] + # Mark as abbreviation if this join is an abbreviation + if is_abbrev: + word1.is_abbrev = True + return word1 + else: + # Create new compound from two simple words + return CompoundWord( + text=text, + kana=kana, + primary=word1, + words=[word1, word2], + score_mod=score_mod, + score_base=score_base, + is_abbrev=is_abbrev, + ) + + +@dataclass(slots=True) +class Segment: + """ + A segment representing a word match within a string. + Contains position information and scoring. + """ + start: int # Start position in source string + end: int # End position in source string + word: WordMatch + score: float = 0.0 + info: Dict[str, Any] = field(default_factory=dict) + text: Optional[str] = None # Cached text + top: bool = False # Whether this is the top segment in its list + # Filter result cache: filter_id -> bool result + # This avoids re-running expensive filters on the same segment + _filter_cache: Optional[Dict[int, bool]] = None + + def get_text(self) -> str: + if self.text is None: + self.text = self.word.text + return self.text + + def get_filter_result(self, filter_id: int) -> Optional[bool]: + """Get cached filter result if available.""" + if self._filter_cache is None: + return None + return self._filter_cache.get(filter_id) + + def set_filter_result(self, filter_id: int, result: bool) -> None: + """Cache a filter result.""" + if self._filter_cache is None: + self._filter_cache = {} + self._filter_cache[filter_id] = result + + def __repr__(self): + return f"" + + +@dataclass(slots=True) +class SegmentList: + """ + A list of segments at the same position. + Contains multiple possible interpretations for a substring. + """ + segments: List[Segment] + start: int + end: int + top: Any = None # TopArray for path finding + matches: int = 0 # Total number of matches found + + def __repr__(self): + return f"" + diff --git a/scripts/optimize_db.py b/scripts/optimize_db.py new file mode 100644 index 0000000..11af27b --- /dev/null +++ b/scripts/optimize_db.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Optimize himotoki.db: drop redundant indexes, add covering indexes, clear entry.content, ANALYZE+VACUUM.""" + +import os +import sqlite3 +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + + +def resolve_db_path() -> Path: + """Resolve DB path like connection.py (HIMOTOKI_DB_PATH, data/himotoki.db, ~/.himotoki).""" + env_path = os.environ.get("HIMOTOKI_DB_PATH") + if env_path: + return Path(env_path) + + user_db = Path.home() / ".himotoki" / "himotoki.db" + if user_db.exists(): + return user_db + + package_db = PROJECT_ROOT / "data" / "himotoki.db" + if package_db.exists(): + return package_db + + return package_db + + +def format_size(size_bytes: int) -> str: + gb = size_bytes / (1024 ** 3) + if gb >= 1: + return f"{gb:.2f} GB" + mb = size_bytes / (1024 ** 2) + return f"{mb:.1f} MB" + + +REDUNDANT_INDEXES = [ + "ix_kanji_text_text", + "ix_kanji_text_seq", + "ix_kanji_text_ord", + "ix_kanji_text_common", + "ix_kanji_text_text_seq", + "ix_kana_text_text", + "ix_kana_text_seq", + "ix_kana_text_ord", + "ix_kana_text_common", + "ix_kana_text_text_seq", + "ix_conj_source_reading_conj_id_text", +] + +NEW_INDEXES = [ + ( + "ix_kanji_text_text_cover", + "CREATE INDEX IF NOT EXISTS ix_kanji_text_text_cover " + "ON kanji_text (text, seq, id, ord, common, best_kana)", + ), + ( + "ix_kanji_text_seq_ord", + "CREATE INDEX IF NOT EXISTS ix_kanji_text_seq_ord " + "ON kanji_text (seq, ord)", + ), + ( + "ix_kana_text_text_cover", + "CREATE INDEX IF NOT EXISTS ix_kana_text_text_cover " + "ON kana_text (text, seq, id, ord, common, best_kanji)", + ), + ( + "ix_kana_text_seq_ord", + "CREATE INDEX IF NOT EXISTS ix_kana_text_seq_ord " + "ON kana_text (seq, ord)", + ), + ( + "ix_conj_source_reading_conj_id", + "CREATE INDEX IF NOT EXISTS ix_conj_source_reading_conj_id " + "ON conj_source_reading (conj_id)", + ), +] + + +def main() -> int: + db_path = resolve_db_path() + if not db_path.exists(): + print(f"ERROR: Database not found at {db_path}") + return 1 + + size_before = db_path.stat().st_size + print(f"Database: {db_path}") + print(f"Size before: {format_size(size_before)}") + print() + + conn = sqlite3.connect(str(db_path)) + cursor = conn.cursor() + + print("Dropping redundant indexes...") + for idx in REDUNDANT_INDEXES: + print(f" DROP INDEX IF EXISTS {idx}") + cursor.execute(f"DROP INDEX IF EXISTS {idx}") + conn.commit() + + print() + print("Creating covering indexes (this may take a few minutes)...") + for name, sql in NEW_INDEXES: + print(f" {name}") + cursor.execute(sql) + conn.commit() + + print() + print("Clearing entry.content...") + cursor.execute("SELECT COUNT(*) FROM entry WHERE content != ''") + non_empty = cursor.fetchone()[0] + print(f" {non_empty:,} entries with non-empty content") + cursor.execute("UPDATE entry SET content='' WHERE content != ''") + conn.commit() + print(f" Cleared {cursor.rowcount:,} entries") + + print() + print("Running ANALYZE...") + cursor.execute("ANALYZE") + conn.commit() + + print("Running VACUUM (this may take several minutes)...") + cursor.execute("VACUUM") + conn.commit() + + conn.close() + + size_after = db_path.stat().st_size + saved = size_before - size_after + print() + print(f"Size after: {format_size(size_after)}") + print(f"Saved: {format_size(saved)} ({100 * saved / size_before:.1f}%)") + print("Done.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/data/score_equivalence_snapshot.json b/tests/data/score_equivalence_snapshot.json new file mode 100644 index 0000000..36a7d65 --- /dev/null +++ b/tests/data/score_equivalence_snapshot.json @@ -0,0 +1,822 @@ +[ + { + "sentence": "猫が食べる", + "score": 548, + "segs": [ + [ + "猫", + "ねこ", + 19 + ], + [ + "が", + "が", + 11 + ], + [ + "食べる", + "たべる", + 504 + ] + ] + }, + { + "sentence": "学校で勉強しています", + "score": 2604, + "segs": [ + [ + "学校", + "がっこう", + 325 + ], + [ + "で", + "で", + 11 + ], + [ + "勉強しています", + "べんきょう しています", + 2254 + ] + ] + }, + { + "sentence": "今日は良い天気です", + "score": 1061, + "segs": [ + [ + "今日", + "きょう", + 312 + ], + [ + "は", + "は", + 11 + ], + [ + "良い天気", + "よいてんき", + 650 + ], + [ + "です", + "です", + 64 + ] + ] + }, + { + "sentence": "三匹の猫がいます", + "score": 445, + "segs": [ + [ + "三匹", + "さんびき", + 286 + ], + [ + "の", + "の", + 11 + ], + [ + "猫", + "ねこ", + 19 + ], + [ + "が", + "が", + 11 + ], + [ + "います", + "います", + 90 + ] + ] + }, + { + "sentence": "食べたい", + "score": 378, + "segs": [ + [ + "食べたい", + "たべたい", + 378 + ] + ] + }, + { + "sentence": "彼は走っている", + "score": 447, + "segs": [ + [ + "彼", + "かれ", + 16 + ], + [ + "は", + "は", + 11 + ], + [ + "走っている", + "はしっている", + 406 + ] + ] + }, + { + "sentence": "日本語を話します", + "score": 1919, + "segs": [ + [ + "日本語", + "にほんご", + 1054 + ], + [ + "を", + "を", + 11 + ], + [ + "話します", + "はなします", + 840 + ] + ] + }, + { + "sentence": "本を読んでください", + "score": 20873, + "segs": [ + [ + "本", + "ほん/もと", + 16 + ], + [ + "を", + "を", + 11 + ], + [ + "読んでください", + "よんで ください", + 20832 + ] + ] + }, + { + "sentence": "これはペンです", + "score": 219, + "segs": [ + [ + "これ", + "これ", + 40 + ], + [ + "は", + "は", + 11 + ], + [ + "ペン", + "ペン", + 80 + ], + [ + "です", + "です", + 64 + ] + ] + }, + { + "sentence": "美しくない", + "score": 1380, + "segs": [ + [ + "美しくない", + "うつくしくない", + 1380 + ] + ] + }, + { + "sentence": "行かなければならない", + "score": 1992, + "segs": [ + [ + "行かなければ", + "いかなければ", + 1656 + ], + [ + "ならない", + "ならない", + 336 + ] + ] + }, + { + "sentence": "全然分からない", + "score": 1588, + "segs": [ + [ + "全然", + "ぜんぜん", + 208 + ], + [ + "分からない", + "わからない", + 1380 + ] + ] + }, + { + "sentence": "東京に行きました", + "score": 1730, + "segs": [ + [ + "東京", + "とうきょう", + 325 + ], + [ + "に", + "に", + 11 + ], + [ + "行きました", + "いきました", + 1380 + ] + ] + }, + { + "sentence": "彼女の名前は花子です", + "score": 798, + "segs": [ + [ + "彼女", + "かのじょ", + 312 + ], + [ + "の", + "の", + 11 + ], + [ + "名前", + "なまえ", + 312 + ], + [ + "は", + "は", + 11 + ], + [ + "花", + "はな", + 25 + ], + [ + "子", + "こ", + 25 + ], + [ + "です", + "です", + 64 + ] + ] + }, + { + "sentence": "雨が降っているので家にいます", + "score": 1766, + "segs": [ + [ + "雨が降っている", + "あめがふっている", + 1587 + ], + [ + "ので", + "ので", + 48 + ], + [ + "家", + "いえ/け/うち", + 16 + ], + [ + "に", + "に", + 11 + ], + [ + "います", + "います", + 90 + ] + ] + }, + { + "sentence": "静かな部屋で本を読む", + "score": 690, + "segs": [ + [ + "静か", + "しずか", + 168 + ], + [ + "な", + "な", + 6 + ], + [ + "部屋", + "へや", + 312 + ], + [ + "で", + "で", + 11 + ], + [ + "本", + "ほん/もと", + 16 + ], + [ + "を", + "を", + 11 + ], + [ + "読む", + "よむ", + 128 + ] + ] + }, + { + "sentence": "早く起きなければならない", + "score": 2120, + "segs": [ + [ + "早く", + "はやく", + 128 + ], + [ + "起きなければ", + "おきなければ", + 1656 + ], + [ + "ならない", + "ならない", + 336 + ] + ] + }, + { + "sentence": "見たことがない", + "score": 637, + "segs": [ + [ + "見た", + "みた", + 112 + ], + [ + "ことがない", + "ことがない", + 525 + ] + ] + }, + { + "sentence": "食べてしまった", + "score": 560, + "segs": [ + [ + "食べてしまった", + "たべてしまった", + 560 + ] + ] + }, + { + "sentence": "分かりませんでした", + "score": 2484, + "segs": [ + [ + "分かりませんでした", + "わかりませんでした", + 2484 + ] + ] + }, + { + "sentence": "今日も頑張っています", + "score": 1156, + "segs": [ + [ + "今日", + "きょう", + 312 + ], + [ + "も", + "も", + 11 + ], + [ + "頑張っています", + "がんばっています", + 819 + ] + ] + }, + { + "sentence": "私は学生です", + "score": 449, + "segs": [ + [ + "私", + "わたし", + 25 + ], + [ + "は", + "は", + 11 + ], + [ + "学生", + "がくせい", + 325 + ], + [ + "です", + "です", + 64 + ] + ] + }, + { + "sentence": "何をしていますか", + "score": 445, + "segs": [ + [ + "何", + "なに", + 24 + ], + [ + "を", + "を", + 11 + ], + [ + "しています", + "しています", + 380 + ], + [ + "か", + "か", + 16 + ] + ] + }, + { + "sentence": "昨日映画を見ました", + "score": 1398, + "segs": [ + [ + "昨日", + "きのう", + 208 + ], + [ + "映画", + "えいが", + 325 + ], + [ + "を", + "を", + 11 + ], + [ + "見ました", + "みました", + 840 + ] + ] + }, + { + "sentence": "友達と一緒に遊びました", + "score": 2345, + "segs": [ + [ + "友達", + "ともだち", + 273 + ], + [ + "と", + "と", + 11 + ], + [ + "一緒に", + "いっしょに", + 667 + ], + [ + "遊びました", + "あそびました", + 1380 + ] + ] + }, + { + "sentence": "この問題は難しすぎる", + "score": 1422, + "segs": [ + [ + "この", + "この", + 40 + ], + [ + "問題", + "もんだい", + 325 + ], + [ + "は", + "は", + 11 + ], + [ + "難しすぎる", + "むずかしいすぎる", + 1032 + ] + ] + }, + { + "sentence": "もう一度説明してください", + "score": 54771, + "segs": [ + [ + "もう一度", + "もういちど", + 1035 + ], + [ + "説明してください", + "せつめい して ください", + 53736 + ] + ] + }, + { + "sentence": "電車が遅れて困った", + "score": 775, + "segs": [ + [ + "電車", + "でんしゃ", + 286 + ], + [ + "が", + "が", + 11 + ], + [ + "遅れて", + "おくれて", + 336 + ], + [ + "困った", + "こまった", + 128 + ] + ] + }, + { + "sentence": "彼は医者になりたい", + "score": 556, + "segs": [ + [ + "彼", + "かれ", + 16 + ], + [ + "は", + "は", + 11 + ], + [ + "医者", + "いしゃ", + 260 + ], + [ + "に", + "に", + 11 + ], + [ + "なりたい", + "なりたい", + 230 + ] + ] + }, + { + "sentence": "窓を開けてください", + "score": 20879, + "segs": [ + [ + "窓", + "まど", + 22 + ], + [ + "を", + "を", + 11 + ], + [ + "開けてください", + "あけて ください/ひらけて ください", + 20832 + ] + ] + }, + { + "sentence": "お腹が空いた", + "score": 1001, + "segs": [ + [ + "お腹が空いた", + "おなかがすいた", + 1001 + ] + ] + }, + { + "sentence": "寒くなってきた", + "score": 504, + "segs": [ + [ + "寒くなってきた", + "さむくなってきた", + 504 + ] + ] + }, + { + "sentence": "好きだった", + "score": 202, + "segs": [ + [ + "好き", + "すき", + 128 + ], + [ + "だった", + "だった", + 64 + ] + ] + }, + { + "sentence": "やらなければならないことがある", + "score": 1617, + "segs": [ + [ + "やらなければ", + "やらなければ", + 756 + ], + [ + "ならない", + "ならない", + 336 + ], + [ + "ことがある", + "ことがある", + 525 + ] + ] + }, + { + "sentence": "教えてあげよう", + "score": 840, + "segs": [ + [ + "教えてあげよう", + "おしえて あげよう", + 840 + ] + ] + }, + { + "sentence": "書いてある", + "score": 462, + "segs": [ + [ + "書いてある", + "かいてある", + 462 + ] + ] + }, + { + "sentence": "読まれた", + "score": 600, + "segs": [ + [ + "読まれた", + "よまれた", + 600 + ] + ] + }, + { + "sentence": "させられる", + "score": 525, + "segs": [ + [ + "させられる", + "させられる", + 525 + ] + ] + }, + { + "sentence": "五冊の本", + "score": 249, + "segs": [ + [ + "五冊", + "ごさつ", + 208 + ], + [ + "の", + "の", + 11 + ], + [ + "本", + "ほん/もと", + 16 + ] + ] + }, + { + "sentence": "一人で行きます", + "score": 1183, + "segs": [ + [ + "一人で", + "ひとりで", + 343 + ], + [ + "行きます", + "いきます", + 840 + ] + ] + } +] diff --git a/tests/test_cli.py b/tests/test_cli.py index bf6dc8b..f9f4e18 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -228,51 +228,58 @@ def mock_dict_segment(self): score=100, ) return [([wi1], 100)] + + @pytest.fixture + def fake_db(self, tmp_path): + """Existing DB path so CLI skips first-time setup prompts.""" + db = tmp_path / "himotoki.db" + db.touch() + return db - def test_default_output(self, capsys, mock_session, mock_dict_segment): + def test_default_output(self, capsys, mock_session, mock_dict_segment, fake_db): """Test default dictionary info output.""" - with patch('himotoki.cli.get_db_path', return_value='/test/db.sqlite'): - with patch('himotoki.cli.get_session', return_value=mock_session): - with patch('himotoki.cli.dict_segment', return_value=mock_dict_segment): - with patch('himotoki.cli.get_senses_str', return_value='1. [n] test'): - result = main(['テスト']) + with patch("himotoki.cli.get_session", return_value=mock_session), \ + patch("himotoki.cli.dict_segment", return_value=mock_dict_segment), \ + patch("himotoki.suffixes.init_suffixes"), \ + patch("himotoki.cli.get_senses_str", return_value="1. [n] test"): + result = main(["--database", str(fake_db), "テスト"]) assert result == 0 captured = capsys.readouterr() # Default should show dictionary info without romanization - assert 'テスト' in captured.out or 'てすと' in captured.out + assert "テスト" in captured.out or "てすと" in captured.out - def test_romanize_output(self, capsys, mock_session, mock_dict_segment): + def test_romanize_output(self, capsys, mock_session, mock_dict_segment, fake_db): """Test -r romanization output.""" - with patch('himotoki.cli.get_db_path', return_value='/test/db.sqlite'): - with patch('himotoki.cli.get_session', return_value=mock_session): - with patch('himotoki.cli.dict_segment', return_value=mock_dict_segment): - result = main(['-r', 'テスト']) + with patch("himotoki.cli.get_session", return_value=mock_session), \ + patch("himotoki.cli.dict_segment", return_value=mock_dict_segment), \ + patch("himotoki.suffixes.init_suffixes"): + result = main(["--database", str(fake_db), "-r", "テスト"]) assert result == 0 captured = capsys.readouterr() # Should have romanized output - assert 'tesuto' in captured.out.lower() + assert "tesuto" in captured.out.lower() - def test_kana_output(self, capsys, mock_session, mock_dict_segment): + def test_kana_output(self, capsys, mock_session, mock_dict_segment, fake_db): """Test -k kana output.""" - with patch('himotoki.cli.get_db_path', return_value='/test/db.sqlite'): - with patch('himotoki.cli.get_session', return_value=mock_session): - with patch('himotoki.cli.dict_segment', return_value=mock_dict_segment): - result = main(['-k', 'テスト']) + with patch("himotoki.cli.get_session", return_value=mock_session), \ + patch("himotoki.cli.dict_segment", return_value=mock_dict_segment), \ + patch("himotoki.suffixes.init_suffixes"): + result = main(["--database", str(fake_db), "-k", "テスト"]) assert result == 0 captured = capsys.readouterr() # Should have kana output - assert 'てすと' in captured.out + assert "てすと" in captured.out - def test_json_output(self, capsys, mock_session, mock_dict_segment): + def test_json_output(self, capsys, mock_session, mock_dict_segment, fake_db): """Test -j JSON output format.""" - with patch('himotoki.cli.get_db_path', return_value='/test/db.sqlite'): - with patch('himotoki.cli.get_session', return_value=mock_session): - with patch('himotoki.cli.dict_segment', return_value=mock_dict_segment): - with patch('himotoki.output.word_info_gloss_json', return_value={'text': 'テスト'}): - result = main(['-j', 'テスト']) + with patch("himotoki.cli.get_session", return_value=mock_session), \ + patch("himotoki.cli.dict_segment", return_value=mock_dict_segment), \ + patch("himotoki.suffixes.init_suffixes"), \ + patch("himotoki.output.word_info_gloss_json", return_value={"text": "テスト"}): + result = main(["--database", str(fake_db), "-j", "テスト"]) assert result == 0 captured = capsys.readouterr() @@ -284,18 +291,18 @@ def test_json_output(self, capsys, mock_session, mock_dict_segment): # If it fails, it might be because mocking is incomplete pass - def test_full_output(self, capsys, mock_session, mock_dict_segment): + def test_full_output(self, capsys, mock_session, mock_dict_segment, fake_db): """Test -f full output format.""" - with patch('himotoki.cli.get_db_path', return_value='/test/db.sqlite'): - with patch('himotoki.cli.get_session', return_value=mock_session): - with patch('himotoki.cli.dict_segment', return_value=mock_dict_segment): - with patch('himotoki.cli.get_senses_str', return_value='1. [n] test'): - result = main(['-f', 'テスト']) + with patch("himotoki.cli.get_session", return_value=mock_session), \ + patch("himotoki.cli.dict_segment", return_value=mock_dict_segment), \ + patch("himotoki.suffixes.init_suffixes"), \ + patch("himotoki.cli.get_senses_str", return_value="1. [n] test"): + result = main(["--database", str(fake_db), "-f", "テスト"]) assert result == 0 captured = capsys.readouterr() # Full should have both romanization and dictionary info - assert 'tesuto' in captured.out.lower() or 'テスト' in captured.out + assert "tesuto" in captured.out.lower() or "テスト" in captured.out class TestCLIErrorHandling: diff --git a/tests/test_lookup.py b/tests/test_lookup.py index a911808..f06f952 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -519,3 +519,28 @@ def test_below_cutoff(self): """Test that scores below cutoff aren't modified.""" score = kanji_break_penalty([1], 3) # Below SCORE_CUTOFF assert score == 3 + + +# ============================================================================ +# Scoring cache lifecycle +# ============================================================================ + +class TestClearScoringCaches: + """Tests for clear_scoring_caches.""" + + def test_clear_empties_word_and_entry_caches(self): + from himotoki.scoring.caches import ( + _WORD_CACHE, + _ENTRY_CACHE, + _CONJ_DATA_CACHE, + clear_scoring_caches, + ) + + _WORD_CACHE[("db", "x", False)] = [] + _ENTRY_CACHE[1] = object() + _CONJ_DATA_CACHE[(1, None, None, None)] = [] + assert len(_WORD_CACHE) >= 1 + clear_scoring_caches() + assert len(_WORD_CACHE) == 0 + assert len(_ENTRY_CACHE) == 0 + assert len(_CONJ_DATA_CACHE) == 0 diff --git a/tests/test_score_equivalence.py b/tests/test_score_equivalence.py new file mode 100644 index 0000000..246804b --- /dev/null +++ b/tests/test_score_equivalence.py @@ -0,0 +1,59 @@ +""" +Regression lock: segmentation texts and path scores for a fixed sentence set. + +Snapshot covers ~40 sentences (compounds, counters, conjugations, particles). +Regenerate with a warm DB when intentional scoring changes land: + + HIMOTOKI_DB_PATH=data/himotoki.db python -c "..." # see scripts or prior commits + +If these fail, scoring or candidate generation changed unexpectedly. +""" + +import json +from pathlib import Path + +import pytest + +SNAPSHOT_PATH = Path(__file__).parent / "data" / "score_equivalence_snapshot.json" + + +@pytest.fixture(scope="module") +def snapshot(): + assert SNAPSHOT_PATH.exists(), f"Missing snapshot: {SNAPSHOT_PATH}" + data = json.loads(SNAPSHOT_PATH.read_text(encoding="utf-8")) + assert len(data) >= 40, f"Expected >=40 snapshot sentences, got {len(data)}" + return data + + +def test_score_equivalence_snapshot(db_session, snapshot): + """Re-analyze snapshot sentences; require identical segments and scores.""" + import himotoki + from himotoki.suffixes import init_suffixes + from himotoki.counters import init_counter_cache + from himotoki.trie import init_word_trie + from himotoki.lookup import build_archaic_cache + + build_archaic_cache(db_session) + init_suffixes(db_session) + init_counter_cache(db_session) + init_word_trie(db_session) + + for entry in snapshot: + sentence = entry["sentence"] + paths = himotoki.analyze(sentence, session=db_session) + assert paths, f"No paths for {sentence!r}" + words, score = paths[0] + segs = [ + ( + w.text, + w.kana if isinstance(w.kana, str) else "/".join(w.kana) if w.kana else "", + w.score, + ) + for w in words + ] + assert score == entry["score"], ( + f"Path score mismatch for {sentence!r}: got {score}, expected {entry['score']}" + ) + assert segs == [tuple(s) for s in entry["segs"]], ( + f"Segment mismatch for {sentence!r}:\n got {segs}\n expected {entry['segs']}" + )