translation and stats skipped words review

Author: jonbitgood

browserbible/js/lib/stopwords.js

@@ -0,0 +1,88 @@
+// Stop words for word-frequency statistics.
+//
+// Word lists live in content/stopwords/<iso639-3>.json (one flat array per
+// language, served from public/ so they stay out of the JS bundle and new
+// languages can be added without code changes). The English list covers
+// function words, light verbs, contractions, and KJV-era archaic forms;
+// distinctive frequent words (God, Lord, verily, behold, amen) are
+// deliberately left out so they stay visible in the stats.
+
+// textInfo.lang is usually a bare ISO 639-3 code but may be suffixed
+// ("eng-Latn-US") or, from some providers, a 2-letter ISO 639-1 code.
+const LANG_ALIASES = {
+  en: 'eng', es: 'spa', pt: 'por', fr: 'fra', hi: 'hin',
+  ar: 'ara', arb: 'ara', ja: 'jpn', ko: 'kor', zh: 'zho', cmn: 'zho'
+};
+
+export function normalizeLang(lang) {
+  if (!lang) return undefined;
+  const primary = String(lang).toLowerCase().split('-')[0];
+  if (!/^[a-z]{2,3}$/.test(primary)) return undefined;
+  return LANG_ALIASES[primary] ?? primary;
+}
+
+const cache = new Map();
+
+/**
+ * Resolve the stop-word Set for a language, or undefined when no list
+ * exists (missing file, network failure, malformed JSON). Results are
+ * cached per language; concurrent callers share one fetch.
+ */
+export function loadStopwords(lang) {
+  const code = normalizeLang(lang);
+  if (!code) return Promise.resolve(undefined);
+
+  if (!cache.has(code)) {
+    cache.set(code, fetch(`content/stopwords/${code}.json`)
+      .then((response) => (response.ok ? response.json() : undefined))
+      .then((words) => (Array.isArray(words) ? new Set(words) : undefined))
+      .catch(() => undefined));
+  }
+  return cache.get(code);
+}
+
+// Unicode letters plus combining marks (Devanagari matras, Arabic diacritics)
+// with word-internal straight or curly apostrophes; trailing possessive 's is
+// stripped so "God's" counts as "God".
+const WORD_PATTERN = /[\p{L}\p{M}]+(?:['’][\p{L}\p{M}]+)*/gu;
+
+// Languages written without spaces, segmented via Intl.Segmenter (which needs
+// a BCP-47 locale). Everything else splits on the regex above.
+const SEGMENTER_LOCALES = {
+  zho: 'zh', cmn: 'zh', yue: 'zh', jpn: 'ja', tha: 'th', khm: 'km', lao: 'lo', mya: 'my'
+};
+const segmenters = new Map();
+
+// Languages where an apostrophe marks elision (l'Éternel, qu'il), so tokens
+// split there; in English it marks contractions (don't) and stays internal.
+const ELISION_LANGS = new Set(['fra']);
+
+export function tokenizeWords(text, lang) {
+  if (!text) return [];
+  const code = normalizeLang(lang);
+
+  const locale = SEGMENTER_LOCALES[code];
+  if (locale && typeof Intl !== 'undefined' && Intl.Segmenter) {
+    if (!segmenters.has(locale)) {
+      segmenters.set(locale, new Intl.Segmenter(locale, { granularity: 'word' }));
+    }
+    const words = [];
+    for (const seg of segmenters.get(locale).segment(text)) {
+      if (seg.isWordLike && /[\p{L}\p{M}]/u.test(seg.segment)) words.push(seg.segment);
+    }
+    return words;
+  }
+
+  const matches = text.match(WORD_PATTERN) ?? [];
+  const tokens = matches.map((token) => token.replace(/['’]s$/i, ''));
+  if (ELISION_LANGS.has(code)) {
+    return tokens.flatMap((token) => token.split(/['’]/).filter(Boolean));
+  }
+  return tokens;
+}
+
+// Canonical form for counting and stop-word lookup: lowercase with curly
+// apostrophes normalized, so "Don’t" matches the "don't" list entry.
+export function wordKey(token) {
+  return token.toLowerCase().replace(/’/g, "'");
+}

browserbible/js/resources/index.js

@@ -12,5 +12,6 @@ export const AVAILABLE_LANGUAGES = [
   'ur',      // Urdu
   'id',      // Indonesian
   'de',      // German
-  'ja'       // Japanese
+  'ja',      // Japanese
+  'ko'       // Korean
 ];

browserbible/js/windows/StatisticsWindow.js

@@ -5,6 +5,7 @@ import { getApp } from '../core/registry.js';
 import { getText, loadSection } from '../texts/TextLoader.js';
 import { renderWordCloud } from '../lib/SimpleWordCloud.js';
 import { escapeRegExp, highlightTextMatches } from '../lib/textHighlighter.js';
+import { loadStopwords, tokenizeWords, wordKey } from '../lib/stopwords.js';
 
 const INIT_DELAY_MS = 1500;
 const FONT_SIZE_MIN = 9;
@@ -16,26 +17,6 @@ const GREEK_STOPWORDS = ['G2532', 'G3588', 'G846', 'G1722', 'G1519', 'G1537', 'G
 const getTextAsync = (textId) => AsyncHelpers.promisifyWithError(getText, textId);
 const loadSectionAsync = (textInfo, sectionId) => AsyncHelpers.promisifyWithError(loadSection, textInfo, sectionId);
 
-const exclusions = {
-  "es": ["de"],
-  "chs": ["-", ":", ",", "。", "(", ")", "!", ";", "一", "?"],
-  "eng": [
-    "a", "abaft", "aboard", "about", "above", "absent", "across", "afore", "after",
-    "against", "along", "alongside", "amid", "amidst", "among", "amongst", "an",
-    "anenst", "apud", "around", "as", "aside", "astride", "at", "athwart", "atop",
-    "barring", "before", "behind", "below", "beneath", "beside", "besides", "between",
-    "beyond", "but", "by", "circa", "concerning", "despite", "down", "during", "except",
-    "excluding", "failing", "following", "for", "forenenst", "from", "given", "in",
-    "including", "inside", "into", "lest", "like", "minus", "modulo", "near", "next",
-    "notwithstanding", "of", "off", "on", "onto", "opposite", "out", "outside", "over",
-    "pace", "past", "per", "plus", "pro", "qua", "regarding", "round", "sans", "save",
-    "since", "than", "through", "throughout", "till", "to", "toward", "towards", "under",
-    "underneath", "unlike", "until", "unto", "up", "upon", "versus", "via", "with",
-    "within", "without", "worth", "the", "him", "his", "he", "she", "it", "her", "hers",
-    "and", "yet", "that", "was", "were", "be", "being", "been", "had", "its", "i"
-  ]
-};
-
 const byCountDescending = (a, b) => b.count - a.count;
 
 function lerp(start, end, min, max, value) {
@@ -56,6 +37,8 @@ class StatisticsWindowComponent extends BaseWindow {
       lemmaData: [],
       hasLemma: false
     };
+
+    this._wordIndex = new Map();
   }
 
   async render() {
@@ -93,7 +76,7 @@ class StatisticsWindowComponent extends BaseWindow {
         this.startProcess(bibleSettings.data.textid, bibleSettings.data.sectionid);
       } else {
         this.refs.statsMainNode.innerHTML =
-          '<div class="statistics-empty">Open a Bible window to see statistics for its current chapter.</div>';
+          `<div class="statistics-empty">${i18n.t('windows.stats.intro')}</div>`;
       }
     }, INIT_DELAY_MS);
   }
@@ -119,6 +102,7 @@ class StatisticsWindowComponent extends BaseWindow {
 
     this.removeHighlights();
     this._statsEpoch = (this._statsEpoch ?? 0) + 1;
+    this._wordIndex = new Map();
 
     Object.assign(this.state, {
       textid: tid,
@@ -169,9 +153,19 @@ class StatisticsWindowComponent extends BaseWindow {
   }
 
   processLemmaVerse(verse) {
+    const stopwords = this._stopwords;
+
     verse.querySelectorAll('l[s]').forEach((lemma) => {
       const word = lemma.innerHTML;
 
+      // Lemma-tagged translations (e.g. ENGWEB) carry surface text in the
+      // target language; skip occurrences that are entirely stop words.
+      const tokens = tokenizeWords(lemma.textContent, this.state.textInfo.lang);
+      if (stopwords && tokens.length > 0 &&
+          tokens.every((t) => stopwords.has(wordKey(t)))) {
+        return;
+      }
+
       for (const strongs of lemma.getAttribute('s').split(' ')) {
         if (GREEK_STOPWORDS.includes(strongs)) continue;
 
@@ -187,26 +181,19 @@ class StatisticsWindowComponent extends BaseWindow {
   }
 
   processTextVerse(verse) {
-    const { lang } = this.state.textInfo;
-    let verseText = verse.innerHTML.replace(/<.*?>/gi, '');
-
-    if (lang.startsWith('en')) {
-      verseText = verseText.replace(/[^A-Za-z\s]/g, '');
-    }
-
-    const langExclusions = exclusions[lang];
-
-    for (const word of verseText.split(' ')) {
-      if (word === '' || langExclusions?.includes(word.toLowerCase())) continue;
+    const stopwords = this._stopwords;
 
-      const entry = this.state.wordStats.find(
-        (wi) => wi.word.toLowerCase() === word.toLowerCase()
-      );
+    for (const word of tokenizeWords(verse.textContent, this.state.textInfo.lang)) {
+      const key = wordKey(word);
+      if (stopwords?.has(key)) continue;
 
+      const entry = this._wordIndex.get(key);
       if (entry) {
         entry.count++;
       } else {
-        this.state.wordStats.push({ word, count: 1 });
+        const newEntry = { word, count: 1 };
+        this._wordIndex.set(key, newEntry);
+        this.state.wordStats.push(newEntry);
       }
     }
   }
@@ -223,11 +210,15 @@ class StatisticsWindowComponent extends BaseWindow {
     const wordCloudNode = resultsNode.querySelector('.statistics-wordcloud');
 
     try {
-      const content = await loadSectionAsync(this.state.textInfo, this.state.sectionid);
+      const [content, stopwords] = await Promise.all([
+        loadSectionAsync(this.state.textInfo, this.state.sectionid),
+        loadStopwords(this.state.textInfo.lang)
+      ]);
       if (epoch !== this._statsEpoch) {
         resultsNode.remove();
         return;
       }
+      this._stopwords = stopwords;
 
       let contentEl = content;
       if (typeof content === 'string') {
@@ -329,7 +320,7 @@ class StatisticsWindowComponent extends BaseWindow {
 
   async loadLemmaInfo(epoch) {
     const lemmaNodeWrapper = this.createElement(`<div class="statistics-section statistics-rare-words">
-      <h3>Rare Words</h3>
+      <h3>${i18n.t('windows.stats.rarewords')}</h3>
       <div class="statistics-results loading-indicator"></div>
     </div>`);
     this.refs.statsMainNode.appendChild(lemmaNodeWrapper);