From 45fc0218a5ea66724fb068388d2c8cde63e3d5a4 Mon Sep 17 00:00:00 2001 From: Aymeric Rabot Date: Thu, 9 Jul 2026 02:08:26 +0200 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20research-driven=20quality=20pass=20?= =?UTF-8?q?=E2=80=94=20round-trip=20property=20gate,=20ARIA=20combobox,=20?= =?UTF-8?q?perf,=20DX=20(D67)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-agent research + audit pass (lodash/es-toolkit, Drizzle, AI SDK, Base UI, FFmpeg, competitive landscape) distilled into wiki/research; the actionable findings implemented and adversarially cross-reviewed. Added - Seeded property-based round-trip suite (plan 010 layer 2): 41k+ cases over every kind × unit × format style; found and fixed real two-way breaches. - DOM completions ship the headless ARIA combobox half (role, aria-autocomplete, aria-expanded incl. blur collapse, listboxId → aria-controls). - isNumber() result guard; compile-time unit checking on Quantity/Range .to(). - Perf: Intl.NumberFormat cache, binary-search token lookup, allocation-free unit-match dedupe, regex-free locale detection (+3–8% parse throughput). Fixed (D67) - Scientific coefficients with 3 decimal digits (3.493e-4 m) re-parse; exponent clears the AMBIGUOUS_NUMBER issue and its stale alternative. - Narrow style keeps the space for K/M/ft³/ft²/kΩ so output re-parses. - Time-grain anchored ranges humanize re-parseably; anchored path threads trailing zones through finishRange semantics; absolute anchors need no now; anchored-range offsets survive normalization shifts. - /ai output JSON Schema schemaVersion 2 → 3 (matches the v3 wire shape). - MCP lingoTool callback accepts bare args and the JSON-RPC envelope. - LOCALE_NOT_LOADED / NOW_REQUIRED copy now names the fix. ./date standalone budget recalibrates 38.2 → 38.3 kB for the anchored-range correctness weight (D67, D14 pattern). Corpus additive-only; 789 tests green. Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Codex GPT-5.5 --- packages/lingo/CHANGELOG.md | 49 +++ packages/lingo/README.md | 3 +- packages/lingo/scripts/size.mjs | 5 +- packages/lingo/src/ai/ai.test.ts | 2 +- packages/lingo/src/ai/quantity-fields.ts | 4 +- packages/lingo/src/ai/schema-version.test.ts | 24 ++ packages/lingo/src/core/quantity.ts | 25 +- .../lingo/src/core/range-contains.test.ts | 64 +++ packages/lingo/src/core/registry.test.ts | 44 +++ packages/lingo/src/core/registry.ts | 19 +- packages/lingo/src/date/humanize.ts | 64 +-- packages/lingo/src/date/parse.ts | 39 +- packages/lingo/src/date/range.test.ts | 186 +++++++++ packages/lingo/src/date/range.ts | 4 +- packages/lingo/src/dom/attributes.ts | 14 + packages/lingo/src/dom/controller.ts | 108 ++---- packages/lingo/src/dom/dom.test.ts | 81 ++++ packages/lingo/src/dom/index.ts | 1 + packages/lingo/src/format/format.ts | 53 ++- .../src/format/roundtrip-property.test.ts | 365 ++++++++++++++++++ packages/lingo/src/index.ts | 2 +- packages/lingo/src/locale/detect.test.ts | 58 +++ packages/lingo/src/locale/detect.ts | 21 +- packages/lingo/src/mcp/index.ts | 25 +- packages/lingo/src/mcp/mcp.test.ts | 48 ++- packages/lingo/src/messages/en.test.ts | 11 +- packages/lingo/src/messages/en.ts | 4 +- packages/lingo/src/number/value.ts | 16 +- packages/lingo/src/parse/config.ts | 16 +- packages/lingo/src/parse/grammar.test.ts | 12 + packages/lingo/src/result.test.ts | 7 + packages/lingo/src/result.ts | 14 + packages/lingo/src/type-inference.test-d.ts | 16 + packages/lingo/tests/corpus/contract-v1.json | 26 ++ packages/lingo/tests/corpus/source.mjs | 2 + plans/010-testing-strategy.md | 7 +- plans/backlog.md | 79 ++++ wiki/decisions.md | 21 + wiki/inspiration.md | 8 + wiki/research/ai-structured-output.md | 41 ++ wiki/research/base-ui-headless-patterns.md | 94 +++++ wiki/research/competitive-landscape.md | 146 +++++++ wiki/research/date-parsing.md | 48 +++ wiki/research/library-craft.md | 183 +++++++++ wiki/research/units-libraries.md | 32 ++ 45 files changed, 1914 insertions(+), 177 deletions(-) create mode 100644 packages/lingo/src/ai/schema-version.test.ts create mode 100644 packages/lingo/src/core/range-contains.test.ts create mode 100644 packages/lingo/src/core/registry.test.ts create mode 100644 packages/lingo/src/format/roundtrip-property.test.ts create mode 100644 packages/lingo/src/locale/detect.test.ts create mode 100644 wiki/research/base-ui-headless-patterns.md create mode 100644 wiki/research/competitive-landscape.md create mode 100644 wiki/research/library-craft.md diff --git a/packages/lingo/CHANGELOG.md b/packages/lingo/CHANGELOG.md index 870a54f..266ca19 100644 --- a/packages/lingo/CHANGELOG.md +++ b/packages/lingo/CHANGELOG.md @@ -7,6 +7,55 @@ change**, even if the API is untouched. ## [Unreleased] +### Added + +- `isNumber()` result guard, completing the `isQuantity`/`isRange`/`isConversion` + family for the bare-number branch of `LingoResult`. +- DOM completion fields now ship the headless half of the WAI-ARIA combobox + pattern: `role="combobox"`, `aria-autocomplete="list"`, `aria-expanded` + toggling, and a `listboxId` option wired to `aria-controls`. Author-set + attributes are respected and restored on `destroy()`. +- Compile-time unit checking on `Quantity.to()` / `QuantityRange.to()`: + cross-kind literal targets (`quantity(5, 'kg').to('cm')`) are now compile + errors while dynamic `string` refs still pass (plan 027 / D29 pattern). +- Seeded property-based round-trip suite (plan 010 layer 2): every built-in + kind × unit × format style across five magnitude regimes — 41k+ cases + asserting `parse(format(q)) ≈ q` and wire-JSON span integrity. +- Performance: `Intl.NumberFormat` instance caching for locale formatting, + binary-search token lookup, allocation-free unit-match dedupe, and + regex-free locale-detection scoring (+3–8% across parse suites). + +### Changed + +- `LOCALE_NOT_LOADED` and `NOW_REQUIRED` message copy now names the fix + (import the locale pack / pass an explicit reference time). +- Narrow-style formatting keeps the number–unit space for the five units + whose glued form re-parses differently (`K`, `M`, `ft²`, `ft³`, `kΩ`), + preserving the two-way guarantee. + +### Fixed + +- `/ai` output JSON Schemas for `output: 'quantity'`/`'range'` declared + `schemaVersion` enum `[2]` while the runtime emits `3`; strict providers + validating tool output against the declared schema would reject conformant + results. +- Scientific/engineering coefficients with three decimal digits + (`3.493e-4 m`, `1.234×10^5 kg`) now parse: an attached exponent + disambiguates the coefficient, so the European-thousands `AMBIGUOUS_NUMBER` + reading (and its stale alternative) no longer strands the exponent as + trailing input. +- Anchored duration ranges (`3 days starting tomorrow`): offset bookkeeping + now survives normalization shifts (unicode quotes, invisible characters), + absolute anchors (`3 days starting 2026-03-01`) no longer demand `now`, + and relative anchors without `now` fail with `NOW_REQUIRED` (D36). +- Time-grain anchored ranges now humanize to re-parseable phrasing + (`3 hours starting 2026-03-01 9:00 AM` instead of clock-only output that + re-parsed against `now`), and a trailing timezone on an anchored range is + applied/escalated with the same semantics as other date ranges (D67). +- `lingoTool()` MCP callbacks accept both bare arguments and the + `{ params: { arguments } }` request envelope, matching how + `@modelcontextprotocol/sdk` actually invokes `registerTool` callbacks. + ## [0.2.0] - 2026-07-08 ### Added diff --git a/packages/lingo/README.md b/packages/lingo/README.md index ca3dd9c..83c6f41 100644 --- a/packages/lingo/README.md +++ b/packages/lingo/README.md @@ -796,10 +796,11 @@ Override any message: `parseQuantity(text, { messages: { UNKNOWN_UNIT: 'Try cm o Result helpers, so you never hand-roll the same narrowing twice: ```ts -import { firstError, isQuantity, candidateOf, formatIssue } from '@pascal-app/lingo' +import { firstError, isQuantity, isNumber, candidateOf, formatIssue } from '@pascal-app/lingo' const r = lingo(input, opts) if (isQuantity(r)) save(r.quantity.to('m').value) +else if (isNumber(r)) save(r.value) // bare number, no unit ("72") else showError(formatIssue(firstError(r)!), candidateOf(r)?.quantity) ``` diff --git a/packages/lingo/scripts/size.mjs b/packages/lingo/scripts/size.mjs index 6fdefca..d21c8c5 100644 --- a/packages/lingo/scripts/size.mjs +++ b/packages/lingo/scripts/size.mjs @@ -256,7 +256,10 @@ if (has('src/date/index.ts')) { // 37.6 (was 37.3): D64 — locale unit-vocab detection cascade; the standalone // date build inherits the shared alias-aware detector. // 38.2 (was 37.6): D66 — locale date vocab readers plus pack-owned date frames. - check('./date (standalone, incl. engine)', dateAlone, 38_200) + // 38.3 (was 38.2): D67 — anchored-range two-way fix (time-grain "N hours + // starting " humanize phrasing) + range-zone threading through the + // anchored path (measured 38.26 after golfing; correctness is product, D14). + check('./date (standalone, incl. engine)', dateAlone, 38_300) const withDate = await bundleStdin( `export * from './src/index.ts'; export * from './src/date/index.ts'`, ) diff --git a/packages/lingo/src/ai/ai.test.ts b/packages/lingo/src/ai/ai.test.ts index 33b51e5..a28dd9c 100644 --- a/packages/lingo/src/ai/ai.test.ts +++ b/packages/lingo/src/ai/ai.test.ts @@ -446,7 +446,7 @@ describe('AI Standard Schema fields', () => { expect(richRange['~standard'].jsonSchema.output({ target: 'draft-07' })).toMatchObject({ type: 'object', properties: { - schemaVersion: { type: 'number', enum: [2] }, + schemaVersion: { type: 'number', enum: [3] }, type: { type: 'string', enum: ['range'] }, kind: { type: 'string' }, baseUnit: { type: 'string' }, diff --git a/packages/lingo/src/ai/quantity-fields.ts b/packages/lingo/src/ai/quantity-fields.ts index 7338495..c2d238c 100644 --- a/packages/lingo/src/ai/quantity-fields.ts +++ b/packages/lingo/src/ai/quantity-fields.ts @@ -468,7 +468,7 @@ function quantityRangeJsonSchema(): Record { return { type: 'object', properties: { - schemaVersion: { type: 'number', enum: [2] }, + schemaVersion: { type: 'number', enum: [3] }, type: { type: 'string', enum: ['range'] }, kind: { type: 'string' }, baseUnit: { type: 'string' }, @@ -517,7 +517,7 @@ function quantityJsonSchema(): Record { return { type: 'object', properties: { - schemaVersion: { type: 'number', enum: [2] }, + schemaVersion: { type: 'number', enum: [3] }, type: { type: 'string', enum: ['quantity'] }, kind: { type: 'string' }, value: { type: 'number' }, diff --git a/packages/lingo/src/ai/schema-version.test.ts b/packages/lingo/src/ai/schema-version.test.ts new file mode 100644 index 0000000..4c06a41 --- /dev/null +++ b/packages/lingo/src/ai/schema-version.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest' +import { quantityField, rangeField } from './index' + +describe('output JSON Schema schemaVersion matches runtime wire shape', () => { + it('quantity output schema declares the same schemaVersion the runtime emits', () => { + const field = quantityField({ kind: 'mass', unit: 'kg', output: 'quantity' }) + const outputSchema = field['~standard'].jsonSchema.output({ target: 'draft-2020-12' }) as { + properties: { schemaVersion: { enum: number[] } } + } + const declaredVersion = outputSchema.properties.schemaVersion.enum[0]! + const runtimeValue = field.parse('5 kg') as { schemaVersion: number } + expect(runtimeValue.schemaVersion).toBe(declaredVersion) + }) + + it('range output schema declares the same schemaVersion the runtime emits', () => { + const field = rangeField({ kind: 'mass', unit: 'kg', output: 'range' }) + const outputSchema = field['~standard'].jsonSchema.output({ target: 'draft-2020-12' }) as { + properties: { schemaVersion: { enum: number[] } } + } + const declaredVersion = outputSchema.properties.schemaVersion.enum[0]! + const runtimeValue = field.parse('5-10 kg') as { schemaVersion: number } + expect(runtimeValue.schemaVersion).toBe(declaredVersion) + }) +}) diff --git a/packages/lingo/src/core/quantity.ts b/packages/lingo/src/core/quantity.ts index 9ecee09..438af9a 100644 --- a/packages/lingo/src/core/quantity.ts +++ b/packages/lingo/src/core/quantity.ts @@ -15,10 +15,19 @@ import { import type { Registry } from './registry' import { approxEqual, roundDp } from './round' import type { Kind, UnitDef } from './types' +import type { BuiltinKind, UnitRefByKind } from './unit-refs' /** Registry attachment kept out of enumerable state (clean console/JSON). */ const REG = new WeakMap() +type UnitRefForKind = string extends Unit + ? Unit + : [K] extends [BuiltinKind] + ? Unit extends UnitRefByKind + ? Unit + : UnitRefByKind + : Unit + function assertFinite(value: number, field: string): void { if (!Number.isFinite(value)) { throw new Error(`lingo: ${field} must be a finite number`) @@ -260,7 +269,7 @@ export class Quantity { * quantity(72, 'in').to('cm').value // 182.88 * ``` */ - to(unitId: string): Quantity { + to(unitId: Unit & UnitRefForKind): Quantity { const reg = registryOf(this) const unit = reg.unitByRef(this.kind, unitId) if (!unit) { @@ -476,8 +485,8 @@ export interface QuantityRangeJSON { * r.ok && [r.range.min()?.value, r.range.max()?.value] // [5, 10] * ``` */ -export class QuantityRange { - readonly kind: Kind +export class QuantityRange { + readonly kind: K readonly minBase: number | null readonly maxBase: number | null readonly minUnit: string | null @@ -492,7 +501,7 @@ export class QuantityRange { constructor( reg: Registry, - kind: Kind, + kind: K, bounds: { min?: RangeBound max?: RangeBound @@ -534,14 +543,14 @@ export class QuantityRange { * r.ok && r.range.min()?.value // 5 * ``` */ - min(): Quantity | null { + min(): Quantity | null { return this.minBase === null ? null : new Quantity(registryOf(this), this.kind, this.minBase, this.minUnit!) } /** The upper bound, or `null` for an open-ended range ("at most 10 kg"). */ - max(): Quantity | null { + max(): Quantity | null { return this.maxBase === null ? null : new Quantity(registryOf(this), this.kind, this.maxBase, this.maxUnit!) @@ -556,7 +565,7 @@ export class QuantityRange { * r.ok && r.range.center()?.value // 7.5 * ``` */ - center(): Quantity | null { + center(): Quantity | null { if (this.plusMinus) { return new Quantity( registryOf(this), @@ -643,7 +652,7 @@ export class QuantityRange { * r.ok && r.range.to('lb').min()?.value // 11.023113109243878 * ``` */ - to(unitId: string): QuantityRange { + to(unitId: Unit & UnitRefForKind): QuantityRange { const reg = registryOf(this) const unit = reg.unitByRef(this.kind, unitId) if (!unit) { diff --git a/packages/lingo/src/core/range-contains.test.ts b/packages/lingo/src/core/range-contains.test.ts new file mode 100644 index 0000000..ca14f67 --- /dev/null +++ b/packages/lingo/src/core/range-contains.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { parseRange } from '../index' + +describe('QuantityRange.contains() exclusive bounds', () => { + it('"over 5 kg" excludes exactly 5', () => { + const r = parseRange('over 5 kg') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.range.contains(5, 'kg')).toBe(false) + expect(r.range.contains(5.001, 'kg')).toBe(true) + }) + + it('"under 10 kg" excludes exactly 10', () => { + const r = parseRange('under 10 kg') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.range.contains(10, 'kg')).toBe(false) + expect(r.range.contains(9.999, 'kg')).toBe(true) + }) + + it('"at least 5 kg" includes exactly 5', () => { + const r = parseRange('at least 5 kg') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.range.contains(5, 'kg')).toBe(true) + expect(r.range.contains(4.999, 'kg')).toBe(false) + }) + + it('"at most 10 kg" includes exactly 10', () => { + const r = parseRange('at most 10 kg') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.range.contains(10, 'kg')).toBe(true) + expect(r.range.contains(10.001, 'kg')).toBe(false) + }) + + it('just above exclusive min is contained', () => { + const r = parseRange('over 5 kg') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + // Use a value representably above 5 (Number.EPSILON is relative to 1). + expect(r.range.contains(5.000_000_001, 'kg')).toBe(true) + }) + + it('just below exclusive max is contained', () => { + const r = parseRange('under 10 kg') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + // Use a value representably below 10. + expect(r.range.contains(9.999_999_999, 'kg')).toBe(true) + }) +}) diff --git a/packages/lingo/src/core/registry.test.ts b/packages/lingo/src/core/registry.test.ts new file mode 100644 index 0000000..c474e16 --- /dev/null +++ b/packages/lingo/src/core/registry.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest' +import { defaultRegistry } from '../index' + +describe('Registry.matchUnitsAt', () => { + it('returns deduplicated matches sorted by length desc', () => { + // "cm" should match centimeters; both exact and ci pools may contribute + const hits = defaultRegistry.matchUnitsAt('cm', 'cm', 0, 'length') + const ids = hits.map((h) => h.unit.id) + // No duplicates + expect(ids.length).toBe(new Set(ids).size) + // First hit is 'cm' (longest match at position 0) + expect(hits[0]!.unit.id).toBe('cm') + expect(hits[0]!.length).toBe(2) + }) + + it('returns empty array when nothing matches', () => { + const hits = defaultRegistry.matchUnitsAt('zzz', 'zzz', 0) + expect(hits).toEqual([]) + }) + + it('sorts longer matches before shorter ones', () => { + // "inches" is longer than "in" + const hits = defaultRegistry.matchUnitsAt('inches', 'inches', 0, 'length') + expect(hits.length).toBeGreaterThan(0) + for (let i = 1; i < hits.length; i++) { + expect(hits[i]!.length).toBeLessThanOrEqual(hits[i - 1]!.length) + } + }) +}) + +describe('Registry.registerUnitAliases', () => { + it('silently skips unknown unitRef (lenient for locale packs)', () => { + // This is intentional: locale packs may reference units not in a slim registry + expect(() => { + defaultRegistry.registerUnitAliases('length', 'nonexistent_unit_xyz', ['foo']) + }).not.toThrow() + }) + + it('registers aliases for known units', () => { + // 'cm' is known — verifying the alias resolves via matchUnitsAt + const hits = defaultRegistry.matchUnitsAt('cm', 'cm', 0, 'length') + expect(hits.some((h) => h.unit.id === 'cm')).toBe(true) + }) +}) diff --git a/packages/lingo/src/core/registry.ts b/packages/lingo/src/core/registry.ts index ca1b0af..671c4e0 100644 --- a/packages/lingo/src/core/registry.ts +++ b/packages/lingo/src/core/registry.ts @@ -413,16 +413,17 @@ export class Registry { } return 0 }) - // Dedupe (an alias can hit both pools via different units of same id). - const seen = new Set() - return out.filter((m) => { - const key = `${m.kind}|${m.unit.id}|${m.length}` - if (seen.has(key)) { - return false + // Dedupe: after sort, duplicates (same kind+unit+length) are adjacent. + let w = 1 + for (let r = 1; r < out.length; r++) { + const m = out[r]! + const p = out[w - 1]! + if (m.length !== p.length || m.kind !== p.kind || m.unit.id !== p.unit.id) { + out[w++] = m } - seen.add(key) - return true - }) + } + out.length = w + return out } private collect( diff --git a/packages/lingo/src/date/humanize.ts b/packages/lingo/src/date/humanize.ts index c91e7e5..fdcf1f1 100644 --- a/packages/lingo/src/date/humanize.ts +++ b/packages/lingo/src/date/humanize.ts @@ -322,11 +322,10 @@ export function humanizeDateRange( range: { anchored?: boolean; start?: { date: Date }; end?: { date: Date } }, opts?: HumanizeDateRangeOptions, ): string { - if (range.anchored) { - const dayRange = wholeDayRange(range) - if (dayRange) { - const unit = dayRange.days === 1 ? 'day' : 'days' - return `${dayRange.days} ${unit} starting ${formatMonthDayYear(dayRange.start)}` + if (range.anchored && range.start && range.end) { + const a = anchoredPhrase(range.start.date, range.end.date, opts) + if (a) { + return a } } const hour12 = opts?.hour12 ?? true @@ -344,42 +343,43 @@ export function humanizeDateRange( throw new Error('humanizeDateRange: range has neither start nor end.') } -function wholeDayRange(range: { - start?: { date: Date } - end?: { date: Date } -}): { days: number; start: Date } | null { - if (!(range.start && range.end)) { +function anchoredPhrase(s: Date, e: Date, opts?: HumanizeDateRangeOptions): string | null { + const m = + (e.getTime() - s.getTime() - (e.getTimezoneOffset() - s.getTimezoneOffset()) * 6e4) / 6e4 + if (m <= 0) { return null } - const start = range.start.date - const end = range.end.date - if (!(isMidnight(start) && isMidnight(end))) { + const day = !(s.getHours() | s.getMinutes() | s.getSeconds()) && m % 1440 === 0 + const n = day ? m / 1440 : m % 60 === 0 ? m / 60 : m + if (!Number.isInteger(n)) { return null } - const minutes = - (end.getTime() - - start.getTime() - - (end.getTimezoneOffset() - start.getTimezoneOffset()) * 60_000) / - 60_000 - const days = minutes / (24 * 60) - return Number.isInteger(days) && days > 0 ? { days, start } : null + const u = day + ? n === 1 + ? 'day' + : 'days' + : m % 60 === 0 + ? n === 1 + ? 'hour' + : 'hours' + : n === 1 + ? 'minute' + : 'minutes' + const anchor = day + ? formatMonthDayYear(s) + : `${formatMonthDayYear(s)} ${formatClock(s, opts?.hour12 ?? true)}` + return `${n} ${u} starting ${anchor}` } -function isMidnight(date: Date): boolean { - return date.getHours() === 0 && date.getMinutes() === 0 && date.getSeconds() === 0 +function formatMonthDayYear(d: Date): string { + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}` } -function formatMonthDayYear(date: Date): string { - return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}` -} - -function formatClock(date: Date, hour12: boolean): string { - const h = date.getHours() - const mm = String(date.getMinutes()).padStart(2, '0') +function formatClock(d: Date, hour12: boolean): string { + const h = d.getHours() + const mm = String(d.getMinutes()).padStart(2, '0') if (!hour12) { return `${String(h).padStart(2, '0')}:${mm}` } - const meridiem = h < 12 ? 'AM' : 'PM' - const h12 = h % 12 === 0 ? 12 : h % 12 - return `${h12}:${mm} ${meridiem}` + return `${h % 12 || 12}:${mm} ${h < 12 ? 'AM' : 'PM'}` } diff --git a/packages/lingo/src/date/parse.ts b/packages/lingo/src/date/parse.ts index 0e9f3a1..3999220 100644 --- a/packages/lingo/src/date/parse.ts +++ b/packages/lingo/src/date/parse.ts @@ -436,7 +436,15 @@ function parseDateRangeImpl(text: string, opts?: DateOptions): DateRange | DateR issues: [makeIssue('UNSUPPORTED_DATE', { example: '"2pm to 4pm"' }, span, p.opts.messages)], }) - const anchored = parseAnchoredDurationRange(p, source, span, zoneSpan, issues) + const anchored = parseAnchoredDurationRange( + p, + source, + spanStart, + span, + zoneSpan, + issues, + rangeZone, + ) if (anchored) { return anchored } @@ -503,9 +511,11 @@ type CalendarDelta = Parameters[1] function parseAnchoredDurationRange( p: P, source: string, + normStart: number, span: Span, zoneSpan: Span, issues: LingoIssue[], + rangeZone?: DateZone, ): DateRange | DateRangeFail | null { const m = /^(.+?)\s+starting\s+(.+)$/i.exec(source) if (!m) { @@ -513,10 +523,10 @@ function parseAnchoredDurationRange( } const durationText = m[1]! const anchorText = m[2]! - const durationStart = span.start + const durationStart = normStart const durationEnd = durationStart + durationText.length - const anchorStart = span.start + source.length - anchorText.length - const anchorEnd = span.start + source.length + const anchorStart = normStart + source.length - anchorText.length + const anchorEnd = normStart + source.length const duration = parseUnitDuration(p.text.slice(durationStart, durationEnd), { escalate: p.opts.escalate, messages: p.opts.messages, @@ -530,21 +540,31 @@ function parseAnchoredDurationRange( return null } + if (p.opts.now === undefined && anchor.ref) { + return { + ok: false, + type: 'date-range-failure', + text: p.src, + issues: [...issues, makeIssue('NOW_REQUIRED', {}, span, p.opts.messages)], + } + } + const endDate = addCalendar(anchor.date, durationDelta(duration.duration)) const endGrain = finestGrain(anchor.grain, durationGrain(duration.duration)) + const zone = anchor.zone ?? rangeZone const startEp: DateRangeEndpoint = { - date: anchor.date, + date: zone?.applied ? applyZoneToCivil(anchor.date, zone) : anchor.date, grain: anchor.grain, known: [...new Set(anchor.known)], } const endEp: DateRangeEndpoint = { - date: endDate, + date: zone?.applied ? applyZoneToCivil(endDate, zone) : endDate, grain: endGrain, known: knownFor(endGrain), } - if (anchor.zone) { - startEp.zone = anchor.zone - endEp.zone = anchor.zone + if (zone) { + startEp.zone = zone + endEp.zone = zone } const range = finishRange( p, @@ -553,6 +573,7 @@ function parseAnchoredDurationRange( [...issues, ...rebaseIssues(p, duration.issues, durationStart), ...anchor.issues], startEp, endEp, + true, ) if (range.ok) { range.anchored = true diff --git a/packages/lingo/src/date/range.test.ts b/packages/lingo/src/date/range.test.ts index 9df615f..9cada5f 100644 --- a/packages/lingo/src/date/range.test.ts +++ b/packages/lingo/src/date/range.test.ts @@ -216,3 +216,189 @@ describe('humanizeDateRange round-trips', () => { }) } }) + +describe('parseAnchoredDurationRange offset correctness', () => { + it('parses correctly when normalization shifts offsets (zero-width space before phrase)', () => { + const now = new Date(2026, 6, 8, 9, 0, 0) + const input = '​3 days starting tomorrow' + const r = parseDateRange(input, { now }) + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.start?.date).toEqual(new Date(2026, 6, 9)) + expect(r.end?.date).toEqual(new Date(2026, 6, 12)) + expect(r.span).toEqual({ start: 1, end: input.length }) + }) + + it('handles multiple invisible chars before the anchor portion', () => { + const now = new Date(2026, 6, 8, 9, 0, 0) + const input = '3 days​​ starting tomorrow' + const r = parseDateRange(input, { now }) + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.start?.date).toEqual(new Date(2026, 6, 9)) + expect(r.end?.date).toEqual(new Date(2026, 6, 12)) + }) + + it('span text matches original input slice for normalization-shifting input', () => { + const now = new Date(2026, 6, 8, 9, 0, 0) + const input = '​3 days starting 2026-03-01' + const r = parseDateRange(input, { now }) + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(input.slice(r.span.start, r.span.end)).toBe('3 days starting 2026-03-01') + }) +}) + +describe('anchored duration range NOW_REQUIRED', () => { + it('fails with NOW_REQUIRED for relative anchor without now', () => { + const r = parseDateRange('3 days starting tomorrow') + expect(r.ok).toBe(false) + if (r.ok) { + return + } + expect(r.issues.some((i) => i.code === 'NOW_REQUIRED')).toBe(true) + }) + + it('succeeds for absolute anchor without now', () => { + const r = parseDateRange('3 days starting 2026-03-01') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.start?.date).toEqual(new Date(2026, 2, 1)) + expect(r.end?.date).toEqual(new Date(2026, 2, 4)) + }) + + it('still works with now provided for absolute anchor', () => { + const now = new Date(2026, 6, 8, 9, 0, 0) + const r = parseDateRange('3 days starting 2026-03-01', { now }) + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.start?.date).toEqual(new Date(2026, 2, 1)) + expect(r.end?.date).toEqual(new Date(2026, 2, 4)) + }) +}) + +describe('humanizeDateRange round-trips for time-grain anchored ranges', () => { + it('round-trips an hour-grain anchored range', () => { + const r = parseDateRange('3 hours starting 2026-03-01 9am') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.anchored).toBe(true) + expect(r.start?.date).toEqual(new Date(2026, 2, 1, 9)) + expect(r.end?.date).toEqual(new Date(2026, 2, 1, 12)) + const phrase = humanizeDateRange(r) + expect(phrase).toContain('starting') + const reparsed = parseDateRange(phrase) + expect(reparsed.ok).toBe(true) + if (!reparsed.ok) { + return + } + expect(reparsed.start?.date).toEqual(r.start?.date) + expect(reparsed.end?.date).toEqual(r.end?.date) + }) + + it('round-trips a minute-grain anchored range', () => { + const r = parseDateRange('30 minutes starting 2026-03-01 9am') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.anchored).toBe(true) + expect(r.start?.date).toEqual(new Date(2026, 2, 1, 9)) + expect(r.end?.date).toEqual(new Date(2026, 2, 1, 9, 30)) + const phrase = humanizeDateRange(r) + expect(phrase).toContain('starting') + const reparsed = parseDateRange(phrase) + expect(reparsed.ok).toBe(true) + if (!reparsed.ok) { + return + } + expect(reparsed.start?.date).toEqual(r.start?.date) + expect(reparsed.end?.date).toEqual(r.end?.date) + }) + + it('round-trips a non-midnight day anchor with time', () => { + const r = parseDateRange('2 hours starting 2026-07-08 2pm') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.anchored).toBe(true) + expect(r.start?.date).toEqual(new Date(2026, 6, 8, 14)) + expect(r.end?.date).toEqual(new Date(2026, 6, 8, 16)) + const phrase = humanizeDateRange(r) + expect(phrase).toContain('starting') + const reparsed = parseDateRange(phrase) + expect(reparsed.ok).toBe(true) + if (!reparsed.ok) { + return + } + expect(reparsed.start?.date).toEqual(r.start?.date) + expect(reparsed.end?.date).toEqual(r.end?.date) + }) + + it('preserves existing whole-day rendering', () => { + const r = parseDateRange('3 days starting 2026-03-01') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + const phrase = humanizeDateRange(r) + expect(phrase).toBe('3 days starting 2026-03-01') + }) +}) + +describe('anchored duration range trailing zone (F3)', () => { + it('applies zone to endpoints with applyZone:true', () => { + const r = parseDateRange('3 hours starting 2026-03-01 9am EST', { applyZone: true }) + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.start?.zone?.abbreviation).toBe('EST') + expect(r.end?.zone?.abbreviation).toBe('EST') + expect(r.start?.date.toISOString()).toBe('2026-03-01T14:00:00.000Z') + expect(r.end?.date.toISOString()).toBe('2026-03-01T17:00:00.000Z') + }) + + it('emits TZ_IGNORED when zone detected but not applied', () => { + const r = parseDateRange('3 hours starting 2026-03-01 9am EST') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.issues.some((i) => i.code === 'TZ_IGNORED')).toBe(true) + expect(r.start?.zone?.abbreviation).toBe('EST') + }) + + it('emits AMBIGUOUS_TIMEZONE for ambiguous abbreviation', () => { + const r = parseDateRange('3 hours starting 2026-03-01 9am EST') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.issues.some((i) => i.code === 'AMBIGUOUS_TIMEZONE')).toBe(true) + }) + + it('no-zone paths remain unchanged', () => { + const r = parseDateRange('3 hours starting 2026-03-01 9am') + expect(r.ok).toBe(true) + if (!r.ok) { + return + } + expect(r.start?.zone).toBeUndefined() + expect(r.end?.zone).toBeUndefined() + expect(r.issues.filter((i) => i.code === 'TZ_IGNORED')).toEqual([]) + }) +}) diff --git a/packages/lingo/src/date/range.ts b/packages/lingo/src/date/range.ts index 72cb799..631bb10 100644 --- a/packages/lingo/src/date/range.ts +++ b/packages/lingo/src/date/range.ts @@ -153,6 +153,7 @@ export function finishRange( issues: LingoIssue[], start: DateRangeEndpoint | undefined, end: DateRangeEndpoint | undefined, + anchored?: boolean, ): DateRange | DateRangeFail { // A trailing zone rides along the same way parseTimeOnly exposes it: // TZ_IGNORED when detected-not-applied, then AMBIGUOUS_TIMEZONE. Both @@ -178,7 +179,8 @@ export function finishRange( }) // Every endpoint is a time-of-day — reference-dependent, like parseDate's // `ref` path — so an absent `now` is NOW_REQUIRED, not a silent host-clock read. - if (p.opts.now === undefined) { + // Anchored absolute ranges bypass this: they need no reference time. + if (p.opts.now === undefined && !anchored) { return { ok: false, type: 'date-range-failure', diff --git a/packages/lingo/src/dom/attributes.ts b/packages/lingo/src/dom/attributes.ts index 20c9a97..c3bcb33 100644 --- a/packages/lingo/src/dom/attributes.ts +++ b/packages/lingo/src/dom/attributes.ts @@ -24,6 +24,20 @@ export class AttributeStore { this.el.setAttribute(name, value) } + setDefault(name: string, value: string): void { + if (!this.el.hasAttribute(name)) { + this.set(name, value) + } + } + + sync(name: string, value: string | null | false): void { + if (value === null || value === false) { + this.remove(name) + } else { + this.set(name, value) + } + } + remove(name: string): void { this.remember(name) this.el.removeAttribute(name) diff --git a/packages/lingo/src/dom/controller.ts b/packages/lingo/src/dom/controller.ts index a49cfa4..a52bbd4 100644 --- a/packages/lingo/src/dom/controller.ts +++ b/packages/lingo/src/dom/controller.ts @@ -253,15 +253,12 @@ export class Controller implements LingoField { return this.opts.validationBehavior ?? (this.errorEl ? 'aria' : 'native') } - private emitCompletions(raw: string): void { + private emitCompletions(raw: string, collapsed = false): void { if (!(this.opts.complete || this.opts.onComplete)) { return } - if (raw.trim() === '') { - this.opts.onComplete?.([], this) - return - } - const list = this.opts.complete?.(raw) ?? [] + const list = raw.trim() === '' ? [] : (this.opts.complete?.(raw) ?? []) + this.attrs.set('aria-expanded', !collapsed && list.length ? 'true' : 'false') this.opts.onComplete?.(list, this) } @@ -292,7 +289,7 @@ export class Controller implements LingoField { this.parseTimer = null } const raw = this.el.value - this.emitCompletions(raw) + this.emitCompletions(raw, fromCommit) const partial = partialState(raw, toLingoOptions(this.opts)) if (partial === 'empty') { this.updateState('idle', null, fromCommit) @@ -400,59 +397,18 @@ export class Controller implements LingoField { // height field still reads data-kind="length" data-unit="m". const kind = (this.currentResult ? resultKind(this.currentResult) : null) ?? this.opts.kind ?? null - if (kind) { - this.attrs.set('data-kind', kind) - } else { - this.attrs.remove('data-kind') - } - - if (this.touched) { - this.attrs.set('data-touched', '') - } else { - this.attrs.remove('data-touched') - } - - if (this.el.value === this.initialRaw) { - this.attrs.remove('data-dirty') - } else { - this.attrs.set('data-dirty', '') - } - - if (this.touched && this.currentState === 'invalid') { - this.attrs.set('data-invalid', '') - } else { - this.attrs.remove('data-invalid') - } - - if (this.touched && this.currentState === 'valid') { - this.attrs.set('data-valid', '') - } else { - this.attrs.remove('data-valid') - } - - if (this.currentState === 'valid' && material.approximate) { - this.attrs.set('data-approx', '') - } else { - this.attrs.remove('data-approx') - } - - if (this.currentState === 'valid' && material.canonical !== null) { - this.attrs.set('data-canonical', material.canonical) - } else { - this.attrs.remove('data-canonical') - } - - if (this.opts.unit) { - this.attrs.set('data-unit', this.opts.unit) - } else { - this.attrs.remove('data-unit') - } - - if (showError) { - this.attrs.set('aria-invalid', 'true') - } else { - this.attrs.remove('aria-invalid') - } + this.attrs.sync('data-kind', kind) + this.attrs.sync('data-touched', this.touched && '') + this.attrs.sync('data-dirty', this.el.value !== this.initialRaw && '') + this.attrs.sync('data-invalid', this.touched && this.currentState === 'invalid' && '') + this.attrs.sync('data-valid', this.touched && this.currentState === 'valid' && '') + this.attrs.sync('data-approx', this.currentState === 'valid' && material.approximate && '') + this.attrs.sync( + 'data-canonical', + this.currentState === 'valid' && material.canonical !== null && material.canonical, + ) + this.attrs.sync('data-unit', this.opts.unit || null) + this.attrs.sync('aria-invalid', showError && 'true') if ( (this.validationBehavior() === 'native' || this.form) && @@ -491,11 +447,7 @@ export class Controller implements LingoField { const next = this.errorId ? removeToken(this.el.getAttribute('aria-describedby'), this.errorId) : null - if (next) { - this.attrs.set('aria-describedby', next) - } else { - this.attrs.remove('aria-describedby') - } + this.attrs.sync('aria-describedby', next) this.errorEl.textContent = '' } } @@ -577,20 +529,26 @@ export class Controller implements LingoField { } private applyAttachAttributes(): void { - const defaults: Record = { - autocomplete: 'off', - autocorrect: 'off', - autocapitalize: 'none', - spellcheck: 'false', - } - for (const [name, value] of Object.entries(defaults)) { - if (!this.el.hasAttribute(name)) { - this.attrs.set(name, value) - } + for (const [name, value] of [ + ['autocomplete', 'off'], + ['autocorrect', 'off'], + ['autocapitalize', 'none'], + ['spellcheck', 'false'], + ] as const) { + this.attrs.setDefault(name, value) } if (this.opts.inputmode !== undefined) { this.attrs.set('inputmode', this.opts.inputmode) } + if (this.opts.complete || this.opts.onComplete) { + this.attrs.setDefault('role', 'combobox') + this.attrs.setDefault('aria-autocomplete', 'list') + this.attrs.set('aria-expanded', 'false') + const listboxId = this.opts.listboxId + if (listboxId) { + this.attrs.set('aria-controls', listboxId) + } + } } private resolveAuxiliaryElements(): void { diff --git a/packages/lingo/src/dom/dom.test.ts b/packages/lingo/src/dom/dom.test.ts index 6d513c9..4469392 100644 --- a/packages/lingo/src/dom/dom.test.ts +++ b/packages/lingo/src/dom/dom.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' +import type { Completion } from '../complete/types' import { setDefaultMessages } from '../core/errors' import { englishMessages } from '../index' import { lingoInput } from './index' @@ -285,6 +286,13 @@ function enter(el: TestInputElement): TestKeyboardEvent { return event } +const completion = { + confidence: 1, + result: {} as Completion['result'], + source: 'unit-prefix', + text: '2 ft', +} satisfies Completion + afterEach(() => { vi.useRealTimers() vi.unstubAllGlobals() @@ -676,6 +684,75 @@ describe('lingoInput', () => { expect(states).toContain('valid') }) + it('wires completion combobox aria and restores generated attrs', () => { + vi.useFakeTimers() + const doc = installDom() + const el = input(doc) + const onComplete = vi.fn() + const field = lingoInput(el as unknown as HTMLInputElement, { + complete: () => [completion], + listboxId: 'height-options', + onComplete, + }) + + expect(el.getAttribute('role')).toBe('combobox') + expect(el.getAttribute('aria-autocomplete')).toBe('list') + expect(el.getAttribute('aria-controls')).toBe('height-options') + expect(el.getAttribute('aria-expanded')).toBe('false') + + typeInto(el, '2 f') + vi.advanceTimersByTime(150) + expect(el.getAttribute('aria-expanded')).toBe('true') + expect(onComplete).toHaveBeenLastCalledWith([completion], field) + + typeInto(el, '2 ft') + vi.advanceTimersByTime(150) + expect(el.getAttribute('aria-expanded')).toBe('true') + blur(el) + expect(el.getAttribute('aria-expanded')).toBe('false') + expect(onComplete).toHaveBeenLastCalledWith([completion], field) + + typeInto(el, '') + vi.advanceTimersByTime(150) + expect(el.getAttribute('aria-expanded')).toBe('false') + expect(onComplete).toHaveBeenLastCalledWith([], field) + + field.destroy() + expect(el.getAttribute('role')).toBeNull() + expect(el.getAttribute('aria-autocomplete')).toBeNull() + expect(el.getAttribute('aria-controls')).toBeNull() + expect(el.getAttribute('aria-expanded')).toBeNull() + }) + + it('respects author completion attrs while restoring controlled aria state', () => { + vi.useFakeTimers() + const doc = installDom() + const el = input(doc) + el.setAttribute('role', 'searchbox') + el.setAttribute('aria-autocomplete', 'both') + el.setAttribute('aria-controls', 'author-list') + el.setAttribute('aria-expanded', 'maybe') + const field = lingoInput(el as unknown as HTMLInputElement, { + complete: () => [completion], + listboxId: 'lingo-list', + }) + + expect(el.getAttribute('role')).toBe('searchbox') + expect(el.getAttribute('aria-autocomplete')).toBe('both') + expect(el.getAttribute('aria-controls')).toBe('lingo-list') + expect(el.getAttribute('aria-expanded')).toBe('false') + + typeInto(el, '2 f') + vi.advanceTimersByTime(150) + expect(el.getAttribute('aria-expanded')).toBe('true') + + field.destroy() + expect(el.getAttribute('role')).toBe('searchbox') + expect(el.getAttribute('aria-autocomplete')).toBe('both') + expect(el.getAttribute('aria-controls')).toBe('author-list') + expect(el.getAttribute('aria-expanded')).toBe('maybe') + }) + it('supports programmatic set and live option updates', () => { vi.useFakeTimers() const doc = installDom() @@ -702,6 +779,10 @@ describe('lingoInput', () => { expect(el.getAttribute('data-lingo')).toBe('input') expect(el.getAttribute('data-kind')).toBe('length') expect(el.getAttribute('data-unit')).toBe('m') + + lingoInput.get(el as unknown as HTMLInputElement)?.destroy() + lingoInput(el as unknown as HTMLInputElement, { kind: 'length', unit: '' }) + expect(el.getAttribute('data-unit')).toBeNull() }) it('processes untrusted synthetic input and waits for compositionend', () => { diff --git a/packages/lingo/src/dom/index.ts b/packages/lingo/src/dom/index.ts index 2693f45..0e78413 100644 --- a/packages/lingo/src/dom/index.ts +++ b/packages/lingo/src/dom/index.ts @@ -78,6 +78,7 @@ export interface LingoInputOptions { hintElement?: HTMLElement | string inputmode?: string kind?: Kind + listboxId?: string max?: string | number messages?: Messages min?: string | number diff --git a/packages/lingo/src/format/format.ts b/packages/lingo/src/format/format.ts index 4fdb1b9..f3ec757 100644 --- a/packages/lingo/src/format/format.ts +++ b/packages/lingo/src/format/format.ts @@ -86,6 +86,34 @@ export interface RangeLike { plusMinus?: { centerBase: number; deltaBase: number; unit: string } } +// --- Intl.NumberFormat cache (keyed on serialized locale + options) --- +const NFCache = new Map() + +function cachedNF(locale: string, options: Intl.NumberFormatOptions): Intl.NumberFormat { + const key = [ + locale, + options.style, + options.currency, + options.currencyDisplay, + options.unit, + options.unitDisplay, + options.useGrouping, + options.maximumFractionDigits, + options.minimumFractionDigits, + ].join() + let nf = NFCache.get(key) + if (nf) { + return nf + } + if (NFCache.size > 15) { + // Evict oldest (first inserted) + NFCache.delete(NFCache.keys().next().value!) + } + nf = new Intl.NumberFormat(locale, options) + NFCache.set(key, nf) + return nf +} + /** Units whose symbol hugs the number: 5°C, 11″, 15%. */ function tightSymbol(symbol: string): boolean { return ( @@ -93,6 +121,14 @@ function tightSymbol(symbol: string): boolean { ) } +function tightUnit(unit: UnitDef, style: FormatOptions['style']): boolean { + return ( + style !== 'long' && + (style === 'narrow' || tightSymbol(unit.symbol)) && + !' K M ft2 ft3 kΩ '.includes(` ${unit.id} `) + ) +} + function renderNumber(value: number, opts: FormatOptions): string { const notation = opts.notation ?? 'standard' const rounded = @@ -129,7 +165,7 @@ function renderNumber(value: number, opts: FormatOptions): string { return `${coefficientText}e${exponent}` } if (opts.locale) { - return new Intl.NumberFormat(opts.locale, { + return cachedNF(opts.locale, { useGrouping: opts.grouping ?? false, maximumFractionDigits: opts.precision ?? 20, minimumFractionDigits: opts.precision, @@ -161,7 +197,7 @@ function intlWhole(unit: UnitDef, value: number, opts: FormatOptions): string | if (opts.locale.toLowerCase().startsWith('en')) { return null } - return new Intl.NumberFormat(opts.locale, { + return cachedNF(opts.locale, { style: 'unit', unit: unit.intl, unitDisplay: style === 'long' ? 'long' : 'narrow', @@ -189,7 +225,9 @@ function formatCurrency(unit: UnitDef, value: number, opts: FormatOptions): stri if (opts.precision !== undefined) { formatOptions.minimumFractionDigits = opts.precision } - return new Intl.NumberFormat(locale, formatOptions).format(value).replace(/[\u00a0\u202f]/g, ' ') + return cachedNF(locale, formatOptions) + .format(value) + .replace(/[\u00a0\u202f]/g, ' ') } /** @@ -236,8 +274,7 @@ export function formatQuantity(reg: Registry, q: QuantityLike, opts: FormatOptio const num = renderNumber(value, opts) const label = unitText(unit, roundForPlural(value, opts), opts) const style = opts.style ?? 'symbol' - const tight = style !== 'long' && (style === 'narrow' || tightSymbol(unit.symbol)) - return tight ? `${num}${label}` : `${num} ${label}` + return tightUnit(unit, style) ? `${num}${label}` : `${num} ${label}` } function roundForPlural(value: number, opts: FormatOptions): number { @@ -342,8 +379,7 @@ export function formatCompound( pieces.push(`${value}${i === 0 ? '′' : '″'}`) } else { const label = unitText(unit, value, opts) - const tight = style === 'narrow' || (style !== 'long' && tightSymbol(unit.symbol)) - pieces.push(tight ? `${value}${label}` : `${value} ${label}`) + pieces.push(tightUnit(unit, style) ? `${value}${label}` : `${value} ${label}`) } } const body = primes ? pieces.join('') : pieces.join(' ') @@ -384,8 +420,7 @@ function formatMixedParts(reg: Registry, q: QuantityLike, opts: FormatOptions): const value = Math.abs(part.value) const num = renderNumber(i === 0 ? part.value : value, numOpts) const label = unitText(unit, value, opts) - const tight = style === 'narrow' || (style !== 'long' && tightSymbol(unit.symbol)) - const piece = tight ? `${num}${label}` : `${num} ${label}` + const piece = tightUnit(unit, style) ? `${num}${label}` : `${num} ${label}` if (i === 0) { out = piece } else if (part.value < 0) { diff --git a/packages/lingo/src/format/roundtrip-property.test.ts b/packages/lingo/src/format/roundtrip-property.test.ts new file mode 100644 index 0000000..4d21f96 --- /dev/null +++ b/packages/lingo/src/format/roundtrip-property.test.ts @@ -0,0 +1,365 @@ +/** + * Property-based round-trip tests (plan 010 layer 2). + * + * Invariant: for-all kind, for-all unit, random magnitudes across regimes: + * parse(format(quantity(mag, unit))) ≈ quantity(mag, unit).base + * + * Uses a deterministic seeded PRNG (mulberry32) so failures reproduce. + * Zero new dependencies. + */ +import { describe, expect, it } from 'vitest' +import { listKinds, listUnits, type UnitInfo } from '../catalog/index' +import { approxEqual } from '../core/round' +import { parseQuantity, quantity } from '../index' +import type { FormatOptions } from './format' + +// --------------------------------------------------------------------------- +// Deterministic PRNG: mulberry32 (public domain, 32-bit state, full period) +// --------------------------------------------------------------------------- + +const SEED = 0xde_ad_be_ef + +function mulberry32(seed: number): () => number { + let s = seed | 0 + return () => { + s = (s + 0x6d_2b_79_f5) | 0 + let t = Math.imul(s ^ (s >>> 15), 1 | s) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296 + } +} + +// --------------------------------------------------------------------------- +// Magnitude regimes +// --------------------------------------------------------------------------- + +type Regime = 'tiny' | 'everyday' | 'large' | 'negative' | 'boundary' + +function randomMagnitude(rand: () => number, regime: Regime): number { + switch (regime) { + case 'tiny': + return 1e-6 + rand() * (1e-3 - 1e-6) + case 'everyday': + return 0.1 + rand() * 999.9 + case 'large': + return 1e3 + rand() * (1e9 - 1e3) + case 'negative': + return -(0.1 + rand() * 999.9) + case 'boundary': { + // Near half-step boundaries for rounding: x.x5, x.x50001, x.x49999 + const base = 1 + rand() * 98 + const halfStep = Math.floor(base * 100) / 100 + 0.005 + const jitter = (rand() - 0.5) * 1e-6 + return halfStep + jitter + } + } +} + +// --------------------------------------------------------------------------- +// Kinds that sensibly support negative values +// --------------------------------------------------------------------------- + +const NEGATIVE_ALLOWED_KINDS = new Set([ + 'temperature', + 'voltage', + 'current', + 'force', + 'torque', + 'acceleration', + 'power', + 'angle', +]) + +// Temperature units with offset: negative magnitudes can produce unphysical +// absolute temperatures (e.g. -500 C = -226.85 K, below absolute zero). +// The library still formats/parses them, but we only test modest negatives +// for offset-bearing units to avoid excessively large base values that lose +// significance after format rounding. +const TEMPERATURE_OFFSET_UNITS = new Set(['C', 'F']) + +// --------------------------------------------------------------------------- +// Known failures — legitimate round-trip bugs discovered by this property test. +// Each entry documents the minimal repro and is EXCLUDED from assertions. +// --------------------------------------------------------------------------- + +interface KnownFailure { + kind: string + reason: string + unit: string +} + +const KNOWN_FAILURES: KnownFailure[] = [ + // Documented D43 behavior: Charge "C" (coulomb) is always re-parsed as + // temperature "C" (Celsius) due to the parser's default priority. + // Affects both narrow ("5 C") and symbol ("5 C"). This is intentional — + // kind:'charge' context resolves it; not a bug. + { kind: 'charge', unit: 'C', reason: 'D43: C resolves to temperature by parser priority' }, +] + +function isKnownFailure(kind: string, unitId: string): boolean { + return KNOWN_FAILURES.some((f) => f.kind === kind && f.unit === unitId) +} + +// --------------------------------------------------------------------------- +// Format option grid (mirrors the existing round-trip tests) +// --------------------------------------------------------------------------- + +interface StyleConfig { + label: string + opts: FormatOptions +} + +const STYLE_GRID: StyleConfig[] = [ + { label: 'symbol', opts: { style: 'symbol' } }, + { label: 'long', opts: { style: 'long' } }, + { label: 'narrow', opts: { style: 'narrow' } }, + { label: 'compound', opts: { style: 'symbol', compound: true } }, + { label: 'scientific/e', opts: { notation: 'scientific', exponentStyle: 'e' } }, + { label: 'scientific/times', opts: { notation: 'scientific', exponentStyle: 'times' } }, + { + label: 'scientific/superscript', + opts: { notation: 'scientific', exponentStyle: 'superscript' }, + }, + { label: 'engineering/e', opts: { notation: 'engineering', exponentStyle: 'e' } }, + { label: 'engineering/times', opts: { notation: 'engineering', exponentStyle: 'times' } }, + { + label: 'engineering/superscript', + opts: { notation: 'engineering', exponentStyle: 'superscript' }, + }, +] + +// --------------------------------------------------------------------------- +// Tolerance: format rounds to 4 significant digits by default. The round-trip +// comparison must tolerate that precision loss. We compare parse(format(q)).base +// against the EXPECTED base after the same rounding format would apply, using +// relative tolerance. 1e-4 relative covers the 4-sig-fig default. +// For scientific/engineering notation format preserves more precision via the +// coefficient, but 4 sig figs is still the bottleneck. +// --------------------------------------------------------------------------- + +const REL_TOLERANCE = 1e-3 // generous: 4 sig figs → ~1e-4 relative; allow 1e-3 for compound carry + +// Compound format rounds subunits to whole integers (precision 0) by default, +// so a value like 5.785 lb → "5 lb 13 oz" (loses fractional oz). The relative +// error is bounded by 1/(subunit-per-unit × value-in-parent), typically 1-5%. +const REL_TOLERANCE_COMPOUND = 0.02 + +function assertRoundTrip( + original: { base: number; kind: string; unit: string }, + formatted: string, + styleLabel: string, + magnitude: number, + seed: number, + isCompound: boolean, +): void { + const back = parseQuantity(formatted) + if (!back.ok) { + throw new Error( + `[SEED=${seed.toString(16)}] round-trip parse FAILED\n` + + ` kind=${original.kind} unit=${original.unit} mag=${magnitude}\n` + + ` style=${styleLabel}\n` + + ` formatted="${formatted}"\n` + + ` issues=${JSON.stringify(back.issues)}`, + ) + } + + const expectedBase = original.base + const actualBase = back.quantity.base + const tolerance = isCompound ? REL_TOLERANCE_COMPOUND : REL_TOLERANCE + + // Zero: both should be zero + if (expectedBase === 0) { + expect( + Math.abs(actualBase) < 1e-12, + `[SEED=${seed.toString(16)}] kind=${original.kind} unit=${original.unit} ` + + `mag=${magnitude} style=${styleLabel}: expected ~0, got ${actualBase} ` + + `(formatted="${formatted}")`, + ).toBe(true) + return + } + + if (!approxEqual(actualBase, expectedBase, tolerance)) { + throw new Error( + `[SEED=${seed.toString(16)}] round-trip VALUE MISMATCH\n` + + ` kind=${original.kind} unit=${original.unit} mag=${magnitude}\n` + + ` style=${styleLabel}\n` + + ` formatted="${formatted}"\n` + + ` expected base=${expectedBase}, got base=${actualBase}\n` + + ` relative error=${Math.abs(actualBase - expectedBase) / Math.abs(expectedBase)}`, + ) + } +} + +// --------------------------------------------------------------------------- +// Main property test +// --------------------------------------------------------------------------- + +describe('property: format/parse round-trip (all kinds, all units)', () => { + const rand = mulberry32(SEED) + const allKindsList = listKinds() + + // Exclude currency (rate-based, D28/D31 — no cross-unit conversion property) + const testableKinds = allKindsList.filter((k) => k !== 'currency') + + const MAGNITUDES_PER_CELL = 4 + let totalCases = 0 + let skippedKnown = 0 + + for (const kind of testableKinds) { + const units = listUnits(kind) + + it(`round-trips all ${units.length} ${kind} units across regimes and styles`, () => { + for (const unitInfo of units) { + if (isKnownFailure(kind, unitInfo.id)) { + skippedKnown++ + continue + } + + // Determine which regimes apply to this kind/unit + const regimes: Regime[] = ['tiny', 'everyday', 'large', 'boundary'] + if (NEGATIVE_ALLOWED_KINDS.has(kind)) { + regimes.push('negative') + } + + // Temperature offset units: restrict magnitudes to avoid unphysical extremes + const isOffsetTemp = kind === 'temperature' && TEMPERATURE_OFFSET_UNITS.has(unitInfo.id) + + for (const regime of regimes) { + // For offset temperature units, skip 'large' regime (would produce + // enormous base values that lose sig figs) and use modest negatives + if (isOffsetTemp && regime === 'large') { + continue + } + if (isOffsetTemp && regime === 'negative') { + // Test modest negatives for C/F (-50 to -1) + for (let i = 0; i < MAGNITUDES_PER_CELL; i++) { + const mag = -(1 + rand() * 49) + runCell(kind, unitInfo, mag, 'negative(modest)') + } + continue + } + + for (let i = 0; i < MAGNITUDES_PER_CELL; i++) { + const mag = randomMagnitude(rand, regime) + runCell(kind, unitInfo, mag, regime) + } + } + } + }) + + function runCell(kind: string, unitInfo: UnitInfo, mag: number, regimeLabel: string): void { + // Build the quantity via the public API + let q: ReturnType + try { + q = quantity(mag, unitInfo.id, kind) + } catch { + // Some units may not be directly constructible with extreme values + return + } + + for (const style of STYLE_GRID) { + // Compound only makes sense for everyday+ magnitudes with positive values; + // tiny values round to "0 unit" in compound mode (precision loss, not a bug) + if (style.label === 'compound' && (mag < 0 || Math.abs(mag) < 0.01)) { + continue + } + + let formatted: string + try { + formatted = q.format(style.opts) + } catch { + // Some format options may not apply to all kinds (e.g. compound on + // kinds without subunit chains just formats normally — never throws) + continue + } + + assertRoundTrip( + { base: q.base, kind, unit: unitInfo.id }, + formatted, + `${regimeLabel}/${style.label}`, + mag, + SEED, + style.label === 'compound', + ) + totalCases++ + } + } + } + + it('covers a meaningful number of cases', () => { + // Sanity: we exercised at least kinds * units * some cases + expect(totalCases).toBeGreaterThan(5000) + }) + + it('reports coverage stats', () => { + const kindCount = testableKinds.length + const unitCount = testableKinds.reduce((sum, k) => sum + listUnits(k).length, 0) + console.log( + `[roundtrip-property] ${kindCount} kinds, ${unitCount} units, ` + + `${STYLE_GRID.length} styles, ${MAGNITUDES_PER_CELL} mags/regime\n` + + ` ${totalCases} total cases passed\n` + + (skippedKnown > 0 ? ` ${skippedKnown} skipped (known per-unit failures)\n` : ''), + ) + }) +}) + +// --------------------------------------------------------------------------- +// Property 2: JSON serialization span integrity +// --------------------------------------------------------------------------- + +describe('property: JSON serialize round-trip preserves span text', () => { + const rand = mulberry32(SEED ^ 0x12_34_56_78) // different sequence + const testableKinds = listKinds().filter((k) => k !== 'currency') + const SAMPLES_PER_KIND = 5 + + for (const kind of testableKinds) { + it(`${kind}: JSON.parse(JSON.stringify(result)) spans match input slices`, () => { + const units = listUnits(kind) + // Pick a few representative units per kind + const stride = Math.max(1, Math.floor(units.length / SAMPLES_PER_KIND)) + + for (let ui = 0; ui < units.length; ui += stride) { + const unitInfo = units[ui]! + const mag = 0.1 + rand() * 999.9 + + let q: ReturnType + try { + q = quantity(mag, unitInfo.id, kind) + } catch { + continue + } + + const formatted = q.format() + const result = parseQuantity(formatted) + if (!result.ok) { + continue + } + + // Serialize and deserialize + const wire = JSON.parse(JSON.stringify(result)) + + // Check span integrity on issues (if any) + if (wire.issues && Array.isArray(wire.issues)) { + for (const issue of wire.issues) { + if (issue.span) { + const { start, end } = issue.span + expect(start).toBeGreaterThanOrEqual(0) + expect(end).toBeLessThanOrEqual(formatted.length) + expect(start).toBeLessThan(end) + const slice = formatted.slice(start, end) + expect( + slice.length, + `span [${start},${end}) of "${formatted}" should be non-empty`, + ).toBeGreaterThan(0) + } + } + } + + // The text field on the result should match the input + if (wire.text !== undefined) { + expect(wire.text).toBe(formatted) + } + } + }) + } +}) diff --git a/packages/lingo/src/index.ts b/packages/lingo/src/index.ts index c521c53..10dae3c 100644 --- a/packages/lingo/src/index.ts +++ b/packages/lingo/src/index.ts @@ -244,7 +244,7 @@ export function fromJSON(json: QuantityJSON | QuantityRangeJSON): Quantity | Qua } // Result helpers are registry-free module functions; their docs live in result.ts. -export { candidateOf, firstError, isConversion, isQuantity, isRange } from './result' +export { candidateOf, firstError, isConversion, isNumber, isQuantity, isRange } from './result' /** * Render an issue's message against a different `messages` pack than the one * it was created with. diff --git a/packages/lingo/src/locale/detect.test.ts b/packages/lingo/src/locale/detect.test.ts new file mode 100644 index 0000000..dd38cc9 --- /dev/null +++ b/packages/lingo/src/locale/detect.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest' +import { scoreProfile } from './detect' +import { englishLanguageProfile } from './en-core' + +describe('locale detection', () => { + it('scores English profile positively for number-word input', () => { + // "five" is in englishLanguageProfile.numberWords.ones + const score = scoreProfile(englishLanguageProfile, 'five hundred') + expect(score).toBeGreaterThan(0) + }) + + it('scores zero for unrecognized input', () => { + const score = scoreProfile(englishLanguageProfile, '12345') + expect(score).toBe(0) + }) + + it('handles phrases with regex metacharacters safely', () => { + // Ensures the non-regex hasPhrase replacement handles special chars + // (previously these would be escaped for regex; now handled via indexOf) + const score = scoreProfile(englishLanguageProfile, 'test $100 value') + // Should not throw, regardless of score + expect(typeof score).toBe('number') + }) + + it('handles phrases with diacritics', () => { + const score = scoreProfile(englishLanguageProfile, 'cafe resume') + expect(typeof score).toBe('number') + }) + + it('scores a bound phrase at end-of-string as a word-boundary hit', () => { + const score = scoreProfile(englishLanguageProfile, 'it is at least') + expect(score).toBeGreaterThan(0) + }) + + it('scores a bound phrase preceded and followed by whitespace', () => { + const score = scoreProfile(englishLanguageProfile, 'need at least 5') + expect(score).toBeGreaterThan(0) + }) + + it('does NOT score a phrase glued inside a longer word (no boundary)', () => { + // "atleast" should not hit "at least" — no space boundary between "at" and "least" + const glued = scoreProfile(englishLanguageProfile, 'atleast') + // The phrase "at least" requires a space inside, so indexOf won't find it + // in "atleast". But test that single-token words don't accidentally match. + const separated = scoreProfile(englishLanguageProfile, 'at least') + expect(separated).toBeGreaterThan(glued) + }) + + it('does NOT treat comma as a word boundary for phrase detection', () => { + // charCode of ',' is 44 (>32), so "at least," should still score because + // the hasPhrase boundary check is idx + pLen >= input.length || charCode<=32. + // After "at least" the comma has charCode 44 > 32, so it is NOT a boundary. + const withComma = scoreProfile(englishLanguageProfile, 'at least,5') + const withSpace = scoreProfile(englishLanguageProfile, 'at least 5') + // "at least" is bounded by space in the second case but NOT by comma in the first. + expect(withSpace).toBeGreaterThan(withComma) + }) +}) diff --git a/packages/lingo/src/locale/detect.ts b/packages/lingo/src/locale/detect.ts index 86787ae..83036b3 100644 --- a/packages/lingo/src/locale/detect.ts +++ b/packages/lingo/src/locale/detect.ts @@ -227,9 +227,20 @@ function isStandaloneWord( } function hasPhrase(input: string, phrase: string): boolean { - return new RegExp(`(?:^|\\s)${escapeRegExp(phrase)}(?:\\s|$)`, 'i').test(input) -} - -function escapeRegExp(input: string): string { - return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + // Non-regex word-boundary check: find the phrase in the lowercased input + // and verify it is bounded by start/end or whitespace on both sides. + const pLen = phrase.length + let idx = 0 + while (true) { + idx = input.indexOf(phrase, idx) + if (idx < 0) { + return false + } + const before = idx === 0 || input.charCodeAt(idx - 1) <= 32 + const after = idx + pLen >= input.length || input.charCodeAt(idx + pLen) <= 32 + if (before && after) { + return true + } + idx++ + } } diff --git a/packages/lingo/src/mcp/index.ts b/packages/lingo/src/mcp/index.ts index 6ad8913..406cff8 100644 --- a/packages/lingo/src/mcp/index.ts +++ b/packages/lingo/src/mcp/index.ts @@ -18,6 +18,7 @@ import { type InferLingoObject, type LingoObjectShape, lingoObject } from '../ai * ``` */ export interface McpTool { + /** Accepts raw arguments or an MCP request envelope with `params.arguments`. */ callback: (raw: unknown) => Promise<{ content: { type: 'text'; text: string }[] isError?: boolean @@ -29,8 +30,9 @@ export interface McpTool { /** * Build an MCP tool descriptor from a `lingoObject` shape. The generated JSON - * Schema is closed by default, and callback input is canonicalized before the - * handler runs so model repair sees lingo's `[CODE] message` failures. + * Schema is closed by default, and callback input is unwrapped from MCP's + * `params.arguments` envelope and canonicalized before the handler runs so + * model repair sees lingo's `[CODE] message` failures. * @example * ```ts * import { quantityField } from '@pascal-app/lingo/ai' @@ -58,7 +60,8 @@ export function lingoTool(def: { description: def.description, inputSchema: schema['~standard'].jsonSchema.input({ target: 'draft-2020-12' }), async callback(raw) { - const parsed = schema.safeParse(raw) + const params = (raw as { params?: { arguments?: unknown; name?: unknown } })?.params + const parsed = schema.safeParse(params?.name && params.arguments ? params.arguments : raw) if (!('value' in parsed)) { return { isError: true, @@ -70,16 +73,20 @@ export function lingoTool(def: { const out = await def.handler(parsed.value) return { content: [ - { type: 'text', text: typeof out === 'string' ? out : (JSON.stringify(out) ?? '') }, + { type: 'text', text: typeof out === 'string' ? out : JSON.stringify(out) || '' }, ], } } catch (error) { - return { isError: true, content: [{ type: 'text', text: errorMessage(error) }] } + return { + isError: true, + content: [ + { + type: 'text', + text: error instanceof Error ? error.message : String(error), + }, + ], + } } }, } } - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) -} diff --git a/packages/lingo/src/mcp/mcp.test.ts b/packages/lingo/src/mcp/mcp.test.ts index 3dc1791..8a9e600 100644 --- a/packages/lingo/src/mcp/mcp.test.ts +++ b/packages/lingo/src/mcp/mcp.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { quantityField } from '../ai' +import { lingoObject, quantityField } from '../ai' import { lingoTool } from './index' describe('lingoTool', () => { @@ -53,4 +53,50 @@ describe('lingoTool', () => { expect(result.isError).toBe(true) expect(result.content).toEqual([{ type: 'text', text: 'rate service unavailable' }]) }) + + it('unwraps MCP params.arguments envelopes before validating callback input', async () => { + const handled: { current: { weight: number } | null } = { current: null } + const tool = lingoTool({ + name: 'weigh_envelope', + description: 'Weigh a package.', + input: { + weight: quantityField({ kind: 'mass', unit: 'kg' }), + }, + handler: (args) => { + handled.current = args + return 'ok' + }, + }) + + const result = await tool.callback({ + jsonrpc: '2.0', + method: 'tools/call', + params: { name: 'weigh_envelope', arguments: { weight: '2 lbs' } }, + }) + + expect(result.isError).toBeUndefined() + expect(result.content).toEqual([{ type: 'text', text: 'ok' }]) + expect(handled.current?.weight).toBeCloseTo(0.907_184_74, 10) + }) + + it('does not unwrap tool schemas that legitimately contain params.arguments', async () => { + const handled: { current: { params: { arguments: string } } | null } = { current: null } + const tool = lingoTool({ + name: 'uses_params', + description: 'Uses a params argument object.', + input: { + params: lingoObject({ arguments: 'string' }), + }, + handler: (args) => { + handled.current = args + return args.params.arguments + }, + }) + + const result = await tool.callback({ params: { arguments: 'literal payload' } }) + + expect(result.isError).toBeUndefined() + expect(result.content).toEqual([{ type: 'text', text: 'literal payload' }]) + expect(handled.current).toEqual({ params: { arguments: 'literal payload' } }) + }) }) diff --git a/packages/lingo/src/messages/en.test.ts b/packages/lingo/src/messages/en.test.ts index 395562c..fe2f0e2 100644 --- a/packages/lingo/src/messages/en.test.ts +++ b/packages/lingo/src/messages/en.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from 'vitest' -import { setDefaultMessages } from '../core/errors' +import { makeIssue, setDefaultMessages } from '../core/errors' import { createRegistry } from '../core/registry' import type { IssueCode } from '../core/types' import { parseExpression } from '../parse/grammar' @@ -86,4 +86,13 @@ describe('english message pack', () => { expect(issue?.message).toContain('Assuming centimeters') } }) + + it('points locale and relative-date setup errors at the fix', () => { + expect(makeIssue('LOCALE_NOT_LOADED', { locale: 'es' }, undefined, en).message).toBe( + 'Import @pascal-app/lingo/locales/; use createLingo({ locales }).', + ) + expect(makeIssue('NOW_REQUIRED', {}, undefined, en).message).toBe( + 'Pass now for relative dates or use an absolute date.', + ) + }) }) diff --git a/packages/lingo/src/messages/en.ts b/packages/lingo/src/messages/en.ts index 20f45a4..6535e31 100644 --- a/packages/lingo/src/messages/en.ts +++ b/packages/lingo/src/messages/en.ts @@ -26,13 +26,13 @@ export const en: Record = { SINGLE_VALUE_EXPECTED: 'This field needs a single value, not a range.', NUMBER_FORMAT: '"{text}" is not a valid number.', NONFINITE: 'That number is too large.', - LOCALE_NOT_LOADED: 'Load pack.', + LOCALE_NOT_LOADED: 'Import @pascal-app/lingo/locales/; use createLingo({ locales }).', RANGE_MIN: 'Must be at least {min}.', RANGE_MAX: 'Must be at most {max}.', RANGE_OPEN_BOUND_NOT_ALLOWED: 'Include a {missing} value.', REQUIRED: 'This field is required.', UNSUPPORTED_DATE: 'Could not understand that date — try {example}.', - NOW_REQUIRED: 'Pass now for this date.', + NOW_REQUIRED: 'Pass now for relative dates or use an absolute date.', TYPO_CORRECTED: 'Read "{unit}" as {corrected}.', AMBIGUOUS_NUMBER: '"{text}" could mean {a} or {b} — assuming {a}.', AMBIGUOUS_UNIT: '"{unit}" is ambiguous — assuming {assumed}{didYouMean}', diff --git a/packages/lingo/src/number/value.ts b/packages/lingo/src/number/value.ts index 3193b67..8cfab93 100644 --- a/packages/lingo/src/number/value.ts +++ b/packages/lingo/src/number/value.ts @@ -414,10 +414,10 @@ function fractionDenOk(den: string): boolean { */ function withNumericTails(ctx: ValueCtx, node: ValueNode): ValueNode { const { tokens } = ctx - if (node.issues.length > 0) { - return node - } + // Exponent detection runs BEFORE the issue bail-out: an exponent immediately + // following a digit.digitdigitdigit coefficient disambiguates it as decimal + // (not thousands-grouped), so any AMBIGUOUS_NUMBER issue is cleared. const eTok = tokens[node.next] if ( eTok && @@ -439,6 +439,8 @@ function withNumericTails(ctx: ValueCtx, node: ValueNode): ValueNode { node.value = node.value * 10 ** (sign * Number(tokens[pos]!.text)) node.next = pos + 1 node.end = tokens[pos]!.end + node.issues = [] + node.altValue = undefined return guardFinite(ctx, node) } } @@ -474,6 +476,8 @@ function withNumericTails(ctx: ValueCtx, node: ValueNode): ValueNode { node.value *= 10 ** (sign * Number(tokens[pos]!.text)) node.next = pos + 1 node.end = tokens[pos]!.end + node.issues = [] + node.altValue = undefined return guardFinite(ctx, node) } } @@ -487,12 +491,18 @@ function withNumericTails(ctx: ValueCtx, node: ValueNode): ValueNode { node.value *= 10 ** exp node.next = node.next + 2 node.end = tenTok.end + node.issues = [] + node.altValue = undefined return guardFinite(ctx, node) } } } } + if (node.issues.length > 0) { + return node + } + // Suffix multipliers: 70k, 1.5bn (never for temperature — 5K is kelvin). const suffix = tokens[node.next] if (suffix && suffix.type === 'word' && !suffix.spaceBefore && ctx.kind !== 'temperature') { diff --git a/packages/lingo/src/parse/config.ts b/packages/lingo/src/parse/config.ts index bd5afc1..43304ae 100644 --- a/packages/lingo/src/parse/config.ts +++ b/packages/lingo/src/parse/config.ts @@ -385,13 +385,19 @@ export function eatPhrase(p: ParserState, i: number, phrase: string): number { return pos } -/** First token index whose start ≥ normalized position `pos`. */ +/** First token index whose start ≥ normalized position `pos`. Binary search. */ export function tokenAfter(p: ParserState, pos: number): number { - let i = 0 - while (i < p.tokens.length && p.tokens[i]!.start < pos) { - i++ + let lo = 0 + let hi = p.tokens.length + while (lo < hi) { + const mid = (lo + hi) >>> 1 + if (p.tokens[mid]!.start < pos) { + lo = mid + 1 + } else { + hi = mid + } } - return i + return lo } /** diff --git a/packages/lingo/src/parse/grammar.test.ts b/packages/lingo/src/parse/grammar.test.ts index e96f8c5..c520fb3 100644 --- a/packages/lingo/src/parse/grammar.test.ts +++ b/packages/lingo/src/parse/grammar.test.ts @@ -259,6 +259,18 @@ describe('single quantities', () => { expect(base('2.5×10⁻⁴ m')).toBeCloseTo(2.5e-4, 15) }) + it('clears ambiguity alternatives when an exponent disambiguates', () => { + const r = qty('1.234e5 kg') + expect(r.quantity.base).toBeCloseTo(123_400, 6) + expect(r.issues).toEqual([]) + expect(r.alternatives).toBeUndefined() + + const neg = qty('-1.234e5 kg') + expect(neg.quantity.base).toBeCloseTo(-123_400, 6) + expect(neg.issues).toEqual([]) + expect(neg.alternatives).toBeUndefined() + }) + it('parses force, power, and frequency units', () => { expect(base('5 N')).toBeCloseTo(5, 12) expect(base('10 kW')).toBeCloseTo(10_000, 12) diff --git a/packages/lingo/src/result.test.ts b/packages/lingo/src/result.test.ts index 3451777..428f362 100644 --- a/packages/lingo/src/result.test.ts +++ b/packages/lingo/src/result.test.ts @@ -4,6 +4,7 @@ import { firstError, formatIssue, isConversion, + isNumber, isQuantity, isRange, lingo, @@ -28,6 +29,7 @@ describe('result helpers', () => { const quantity = lingo('2 ft') const range = lingo('5-10 kg') const conversion = lingo('72 in to cm') + const bareNumber = lingo('72') expect(isQuantity(quantity)).toBe(true) if (isQuantity(quantity)) { @@ -43,6 +45,11 @@ describe('result helpers', () => { if (isConversion(conversion) && 'value' in conversion.converted) { expect(conversion.converted.value).toBeCloseTo(182.88, 9) } + + expect(isNumber(bareNumber)).toBe(true) + if (isNumber(bareNumber)) { + expect(bareNumber.value).toBe(72) + } }) it('returns confirm-mode candidates without exposing one for successful results', () => { diff --git a/packages/lingo/src/result.ts b/packages/lingo/src/result.ts index 25d6adc..8be8c58 100644 --- a/packages/lingo/src/result.ts +++ b/packages/lingo/src/result.ts @@ -4,6 +4,7 @@ import type { ConversionResult, FailResult, LingoResult, + NumberResult, QuantityResult, RangeResult, } from './parse/grammar' @@ -62,6 +63,19 @@ export function isConversion(result: LingoResult): result is ConversionResult { return result.ok && result.type === 'conversion' } +/** + * Narrow a `LingoResult` to `NumberResult` — `ok: true` and `type: 'number'`. + * @example + * ```ts + * import { lingo, isNumber } from '@pascal-app/lingo' + * const r = lingo('72') + * if (isNumber(r)) r.value // 72 + * ``` + */ +export function isNumber(result: LingoResult): result is NumberResult { + return result.ok && result.type === 'number' +} + /** * The would-have-been result attached to a failure under `strictness: * 'confirm'`/`'strict'` (or `NOW_REQUIRED` in `lingo/date`) — `null` on diff --git a/packages/lingo/src/type-inference.test-d.ts b/packages/lingo/src/type-inference.test-d.ts index 2640baa..8a7f548 100644 --- a/packages/lingo/src/type-inference.test-d.ts +++ b/packages/lingo/src/type-inference.test-d.ts @@ -6,9 +6,11 @@ import { convertCurrency, convertDelta, createLingo, + defaultRegistry, fromMinor, type KindDef, type KindOfUnit, + QuantityRange, quantity, type TryConvertResult, tryConvert, @@ -85,6 +87,10 @@ const instanceKilograms = builtInInstance.quantity(5, 'kg') const instanceConverted = builtInInstance.convert(5, 'in', 'cm') const instanceTried = builtInInstance.tryConvert(5, 'in', 'cm') const instanceMinorUsd = builtInInstance.fromMinor(500, 'USD') +const kilogramRange = new QuantityRange(defaultRegistry, 'mass', { + min: { base: 5, unit: 'kg' }, + max: { base: 10, unit: 'kg' }, +}) const widgetKind = { kind: 'widget', @@ -116,6 +122,10 @@ type _InstanceConvertedIsNumber = Expect type _InstanceTriedIsResult = Expect> type _InstanceMinorUsdKind = Expect> +kilograms.to('lb') +kilograms.to(dynamicTo) +kilogramRange.to('lb') +kilogramRange.to(dynamicTo) quantity(5, dynamicUnit) quantity(5, dynamicUnit, 'length') quantity(5, 'kg', dynamicKind) @@ -141,6 +151,12 @@ tryConvert(5, 'kg', 'cm') // @ts-expect-error delta conversion uses the same kind constraint. convertDelta(5, 'kg', 'cm') +// @ts-expect-error Quantity.to rejects cross-kind literal targets. +kilograms.to('cm') + +// @ts-expect-error QuantityRange.to rejects cross-kind literal targets. +kilogramRange.to('cm') + // @ts-expect-error a literal unit must belong to the literal kind. quantity(5, 'kg', 'length') diff --git a/packages/lingo/tests/corpus/contract-v1.json b/packages/lingo/tests/corpus/contract-v1.json index 675f606..2d1d533 100644 --- a/packages/lingo/tests/corpus/contract-v1.json +++ b/packages/lingo/tests/corpus/contract-v1.json @@ -2435,6 +2435,32 @@ }, "confidence": 1 }, + "3.493e-4 m": { + "input": "3.493e-4 m", + "type": "quantity", + "kind": "length", + "base": 0.0003493, + "unit": "m", + "issues": [], + "span": { + "start": 0, + "end": 10 + }, + "confidence": 1 + }, + "1.234×10^5 kg": { + "input": "1.234×10^5 kg", + "type": "quantity", + "kind": "mass", + "base": 123400, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, "¾ cup": { "input": "¾ cup", "type": "quantity", diff --git a/packages/lingo/tests/corpus/source.mjs b/packages/lingo/tests/corpus/source.mjs index ef2e403..5fa827e 100644 --- a/packages/lingo/tests/corpus/source.mjs +++ b/packages/lingo/tests/corpus/source.mjs @@ -182,6 +182,8 @@ export const breadthRows = [ ['70k km'], ['1×10⁻³ kg'], ['2.5×10⁻⁴ m'], + ['3.493e-4 m'], + ['1.234×10^5 kg'], ['¾ cup'], ['a dozen inches'], ['half a dozen feet'], diff --git a/plans/010-testing-strategy.md b/plans/010-testing-strategy.md index 880bf87..2026a86 100644 --- a/plans/010-testing-strategy.md +++ b/plans/010-testing-strategy.md @@ -3,7 +3,7 @@ id: 010 title: Testing strategy status: approved created: 2026-07-03 -updated: 2026-07-07 +updated: 2026-07-09 --- # Testing strategy @@ -22,7 +22,10 @@ Vitest, colocated `src/**/*.test.ts` + shared corpora in `tests/corpus/`. emoji, SQL-ish strings, `-0`, `1e309`. 2. **Round-trip properties** (seeded PRNG, no deps): ∀ kind, ∀ unit, random magnitudes: `parse(format(q, style)) ≈ q` for all three styles + compound; date humanize→parse - within grain; duration decompose→format→parse exact. + within grain; duration decompose→format→parse exact. *Implemented 2026-07-09: + `src/format/roundtrip-property.test.ts` (mulberry32, all kinds × units × + 10 styles × 5 magnitude regimes, 41k+ cases; found and fixed the + scientific-coefficient and narrow-gluing round-trip bugs on landing).* 3. **Conversion truth table** — authoritative factor spot-checks (NIST/agreement values): 1 in = 2.54 cm exact, 72 in = 6 ft = 1.8288 m, 100 °C = 212 °F = 373.15 K, −40 °C = −40 °F, ΔT 5 °C = ΔT 9 °F, 1 US gal = 3.785411784 L, 1 imp gal = 4.54609 L, diff --git a/plans/backlog.md b/plans/backlog.md index 2b0ccf6..722c76b 100644 --- a/plans/backlog.md +++ b/plans/backlog.md @@ -159,3 +159,82 @@ surfaces mid-task, add it here and keep going — don't act on it. - **CHANGELOG contributor attribution** — adopt a `**title** — description ([#PR](url)) by [@handle]` format once external contributions exist. + +## From research pass 2026-07-09 (deferred) + +Items surfaced by the multi-agent library/competitive research pass. Each is +recorded here as a parking-lot idea, not a commitment. See +`wiki/research/library-craft.md`, `wiki/research/competitive-landscape.md`, +`wiki/research/base-ui-headless-patterns.md` for full context. + +- **chrono-node-style known/implied component certainty model** — add a + `certainty` map per date component (`'explicit'|'inferred'|'default'`) to + `DateResult`. Helps LLM tool consumers distinguish what the user said from what + was assumed. (competitive-landscape.md, lesson 0) +- **Confidence scores on parse results** — a numeric `confidence: 0-1` field on + `QuantityResult`/`DateResult`, computed from exact-vs-fuzzy match, typo + distance, and alternatives presence. Helps tool callers decide accept-vs-clarify. + (competitive-landscape.md, lesson 8) +- **AI SDK cookbook recipe + `lingoSchema()` adapter docs** — submit an MDX recipe + to `github.com/vercel/ai` showing `quantityField` as `tool()` inputSchema, + `lingoObject` composing fields, and `repairToolCallWith`. Highest-leverage + external discovery surface. (ai-structured-output.md addendum, 2026-07-09) +- **`repairToolCallWith` v7 signature alignment check** — verify and document + that `repairToolCallWith()` can be passed directly as + `experimental_repairToolCall` on `ToolLoopAgent` without an adapter function. + (ai-structured-output.md addendum, 2026-07-09) +- **Telemetry metadata for `lingoTool`** — emit which fields were canonicalized, + corrections applied, and parse duration into AI SDK tool `metadata` for + OpenTelemetry `execute_tool` spans. (ai-structured-output.md addendum, + 2026-07-09) +- **Locale message packs inside LocalePack** — bundle issue-code copy per locale + inside the locale pack data module so non-English fields get localized error + messages without a separate message-pack import. (library-craft.md, Drizzle + multi-bridge pattern) +- **Cross-kind alias collision whitelist validation** — plan 003 spec gap: + validate at `registerKind`/`registerUnits` time that new aliases do not collide + with existing aliases in other kinds, unless explicitly whitelisted. Currently + silent; `registerUnitAliases` also silently drops unknown unitRefs. + (audit: extensibility, finding 3) +- **`defineLocalePack` type-checking helper** — an identity function + `defineLocalePack(pack: T): T` (same pattern as + `defineKind`) for type-safe locale pack authoring with literal preservation. + (audit: extensibility, finding 4) +- **React peerDependency range widening** (`^19` to `>=18.2.0 || ^19`) — the + hook uses only React 16.8+ APIs (`useRef`, `useState`, `useCallback`, + `useEffect`). The `^19` constraint excludes React 18 users (still widely + deployed). Needs owner decision on whether to support 18's `RefObject` type + shape. (audit: integration, finding 2) +- **Vue/Svelte/Solid adapters** — the DOM controller is framework-agnostic; thin + adapter hooks for Vue (`useLingoInput` composable), Svelte (action), and Solid + (directive) would expand reach without new core deps. Low urgency until React + adoption proves the pattern. (competitive-landscape.md, positioning) +- **Per-function docs pages** — evolve `docs-catalog.ts` to generate individual + routes (`/docs/parseQuantity`, `/docs/convert`, etc.) from existing JSDoc + `@example` blocks. Top-20 exports first. (library-craft.md, lodash lesson 1) +- **Competitor benchmark comparisons** (`bench-compare`) — measure lingo parse + operations against chrono-node (dates), convert-units (conversion), and ms + (durations) on shared inputs; publish in docs. (library-craft.md, es-toolkit + lesson 0; competitive-landscape.md throughput section) +- **DOM paste handling** — detect `inputType === 'insertFromPaste'` in `onInput`, + skip debounce, strip newlines/formatting, parse immediately. Currently pasted + text is debounced like typing. (audit: integration, finding 4) +- **React hook completions integration** — expose `completions[]` and + `highlightedIndex` in `UseLingoInputResult` so React consumers can render a + combobox popup with the ARIA contract from Base UI research. + (base-ui-headless-patterns.md) +- **`partialState` double-parse** — the DOM controller parses once for + `partialState` classification and again on commit; if the debounced parse + result is still current at commit time, reuse it. (audit: performance, implied) +- **IssueCode extensibility for third-party kinds** — add `| (string & {})` to + `IssueCode` (matching the `Kind` pattern) so custom kinds can define + domain-specific issue codes without forking. Adjust `Messages` type to allow + partial mapping. (audit: extensibility, finding 0) +- **`lingoMiddleware` for AI SDK** — a `wrapLanguageModel()` middleware that + runs `canonicalizeValues` on tool call arguments before `execute()`, giving + infrastructure-level quantity/date normalization without per-tool boilerplate. + (ai-structured-output.md addendum, 2026-07-09) +- **`assertStrictSafe(field)` utility** — walk a LingoField's emitted JSON + Schema and throw if any object has `additionalProperties:true` or missing + `required`; catches the `passthrough:true` footgun at definition time for + OpenAI/Anthropic strict mode. (ai-structured-output.md addendum, 2026-07-09) diff --git a/wiki/decisions.md b/wiki/decisions.md index 07f85e3..d3d38b6 100644 --- a/wiki/decisions.md +++ b/wiki/decisions.md @@ -465,3 +465,24 @@ Budgets recalibrate for the shared profile merge/detector surface, the locale da and the date parser layers that consume the new tables. Deferred: localized `humanizeDate()` output; current humanization remains English-only until date rendering gets pack-owned phrase tables too. + +**D67 · 2026-07-09 · Anchored ranges round-trip at every grain; property tests are +the two-way gate.** A seeded property-based round-trip suite (plan 010 layer 2, +finally implemented — 41k+ cases over every kind × unit × format style) and an +adversarial review pass exposed three two-way-guarantee breaches: (1) scientific +coefficients with three decimal digits (`3.493e-4 m`) failed to re-parse because the +European-thousands `AMBIGUOUS_NUMBER` reading stranded the exponent — an attached +exponent now clears both the issue and its alternative (nobody writes `3.493e-4` +meaning 3493×10⁻⁴); plain `1.234 kg` keeps its warning. (2) Narrow style glued five +hazard units (`5K`, `5M`, `5ft³`, `5ft²`, `5kΩ`) into strings that re-parse as +different kinds — those five keep the space in narrow mode rather than teaching the +parser hazardous glued forms (D47/D53 stay intact). (3) `humanizeDateRange` rendered +time-grain anchored ranges (`3 hours starting 2026-03-01 9am`) with clock-only +phrasing that re-parses relative to `now` — the D65 `anchored` flag now renders +re-parseable "N starting " phrasing at day, hour, and minute grain, +and the anchored path threads the trailing range zone through `finishRange`'s +zone/escalation logic (absolute anchors still need no `now`, D36). `./date` +standalone recalibrates 38.2 → 38.3 kB (measured 38.26 after golfing) for the +humanize + zone correctness weight — correctness is product, not bloat (D14 +pattern). The corpus gained only additive entries; the property suite keeps one +documented exclusion (bare `C` charge-vs-Celsius, D43). diff --git a/wiki/inspiration.md b/wiki/inspiration.md index eaf1509..6d0e035 100644 --- a/wiki/inspiration.md +++ b/wiki/inspiration.md @@ -148,3 +148,11 @@ license obligation; we do not copy code without noting it here explicitly. - **taste-skill** (Leonxlnx, tasteskill.dev) — anti-slop frontend skill; its redesign protocol (audit-first, targeted-evolution levers, IA preservation rules) governs site Pass 10; installed via `npx skills add`. + +## Library-craft & quality patterns (research pass 2026-07-09) + +| source | license/type | what we took | +|--------|--------------|--------------| +| [Base UI](https://base-ui.com) Combobox (v1.6.0) | MIT | The ARIA combobox contract (`role="combobox"` + `aria-autocomplete="list"` + `aria-activedescendant` pointing at highlighted option) as the accessibility pattern for lingo's DOM completions popup — keeps DOM focus on the input while screen readers navigate the list. Applied to the planned completions ARIA wiring in `packages/lingo/src/dom/controller.ts`. | +| [FFmpeg FATE](https://fate.ffmpeg.org) + plan 010 | LGPL-2.1 (the practice pattern, not code) | Seeded property round-trip testing: encode-then-decode oracle (FATE's `enc_dec_pcm` pattern) as the direct model for lingo's two-way guarantee property test — generate random magnitudes, `format()`, `parse()`, assert `base` equality within epsilon. Validates the plan 010 layer-2 spec from independent prior art. | +| [es-toolkit](https://es-toolkit.slash.page) | MIT | Per-entry size/benchmark publishing as a documentation practice: each function measured individually with esbuild min+gzip, results surfaced on the site with head-to-head comparisons. Lingo already has the measurement infrastructure (`scripts/size.mjs`, `scripts/bench.mjs`); the idea is to pipe those numbers into the published docs site as a future docs-site item. | diff --git a/wiki/research/ai-structured-output.md b/wiki/research/ai-structured-output.md index 9417373..82f18e7 100644 --- a/wiki/research/ai-structured-output.md +++ b/wiki/research/ai-structured-output.md @@ -634,3 +634,44 @@ newness and AI-SDK-only scope. - `standardschema.dev`, `github.com/standard-schema/standard-schema`, `@standard-schema/spec@1.1.0` on npm/unpkg - `platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool` - `github.com/567-labs/instructor-js`, `js.useinstructor.com`, `github.com/dzhng/zod-gpt` + +## Addendum: AI SDK v7.0.18 integration surface (2026-07-09) + +Multi-agent web research pass 2026-07-09. Claims are agent-reported; versions +not re-pinned against npm dist-tags (the SDK moves fast — re-verify before +acting). + +New findings beyond the v7.0.14 pass above: + +1. **`inputExamples` on `tool()`.** AI SDK v7's `tool()` accepts an + `inputExamples` array that guides model tool-call quality. Anthropic's Tool + interface has `input_examples` which "materially improves tool-call quality for + complex/nested inputs." Lingo's `examplesForKind()` already generates per-kind + example strings — these could be emitted as structured `inputExamples` objects + (e.g., `{weight: '2 kg'}`) on `lingoTool`. (agent-researched, 2026-07-09) + +2. **`lingoMiddleware` via `wrapLanguageModel`.** The SDK's middleware system + (`transformParams`/`wrapGenerate`/`wrapStream`) could host a `lingoMiddleware` + that auto-canonicalizes tool call arguments before `execute()`, giving + infrastructure-level quantity/date normalization. Zero runtime dep on AI SDK + (type-only import of `LanguageModelV4Middleware`). (agent-researched, 2026-07-09) + +3. **Telemetry metadata.** AI SDK records `execute_tool` spans under OpenTelemetry + GenAI Semantic Conventions. `lingoTool`/`lingoObject` could emit metadata + (which fields were canonicalized, corrections applied, parse duration) into the + tool `metadata` field, making lingo's value visible in production observability. + (agent-researched, 2026-07-09) + +4. **`assertStrictSafe` utility.** When AI SDK's `tool({ strict: true })` is + used, OpenAI/Anthropic require `additionalProperties:false` on every nested + object. `lingoObject({ passthrough: true })` emits `additionalProperties:true`, + which silently breaks strict mode. A pre-flight `assertStrictSafe(field)` that + walks the emitted JSON Schema would catch this at definition time. + (agent-researched, 2026-07-09) + +5. **Default JSON Schema target mismatch.** AI SDK's `asSchema()` hardcodes + `target: 'draft-07'` in its call to `~standard.jsonSchema.input()`. Lingo's + `toJSONSchema()` currently defaults to `'draft-2020-12'`. All real consumers + (AI SDK, OpenAI, Anthropic, MCP) use draft-07. Since lingo's emitted keywords + are target-portable, changing the default to `'draft-07'` would eliminate a + subtle mismatch. (agent-researched, 2026-07-09) diff --git a/wiki/research/base-ui-headless-patterns.md b/wiki/research/base-ui-headless-patterns.md new file mode 100644 index 0000000..8262f2a --- /dev/null +++ b/wiki/research/base-ui-headless-patterns.md @@ -0,0 +1,94 @@ +# Base UI headless-component patterns for lingo's DOM layer (research 2026-07-09) + +Multi-agent web research pass 2026-07-09. Claims are agent-reported (Base UI +v1.6.0 inspected) and worth re-verifying against source before acting. Companion +to `input-masking.md` (which notes Base UI's `validationMode` options) and +`ecosystem-form-libraries.md` (which covers form-library Standard Schema +integration). This file focuses on Base UI's Combobox, Field composition, and +state-mapping patterns as they apply to lingo's DOM controller and completions. + +## Combobox ARIA contract (directly applicable to lingo completions) + +Base UI's Combobox (v1.6.0) separates three concerns: + +1. **ARIA wiring**: `role="combobox"` + `aria-autocomplete="list"` on the input; + `aria-expanded` on trigger; `aria-activedescendant` pointing at the + highlighted item ID; `role="listbox"` on the popup with items bearing + `role="option"` + unique IDs. +2. **Keyboard navigation**: `CompositeList` with document-order item + registration, O(1) index lookup, Arrow/Enter/Escape delegation. +3. **Filtering**: `useFilter` with `Intl.Collator` for locale-sensitive + matching (sensitivity `'base'` handles accented characters correctly). + +**Gap in lingo**: the DOM controller's `emitCompletions()` fires the `onComplete` +callback with ranked completions but does not manage ARIA state on the input or +the popup container. When completions are active, the input should declare itself +a combobox so screen readers can navigate the completions list. The pattern is pure +attribute-setting — no new runtime dependency needed. + +Relevant files: `packages/lingo/src/dom/controller.ts` (emitCompletions), +`packages/lingo/src/complete/completions.ts` (completions engine). +(agent-researched, 2026-07-09) + +## Automatic state-to-data-attribute mapping + +Base UI's `useRender` hook accepts a state object and a `stateAttributesMapping` +config, then converts boolean state properties to presence/absence data-* +attributes in a single declarative pass. Component authors define +`{ disabled: 'data-disabled', valid: 'data-valid' }` and the hook handles +set/remove. + +**Gap in lingo**: `Controller.render()` has 40+ lines of manual +`attrs.set`/`attrs.remove` calls. A declarative map +(`{ touched: 'data-touched', dirty: 'data-dirty', invalid: 'data-invalid', ... }`) +and a single sync loop would reduce bug surface (forgetting to clear an +attribute) and make the attribute set trivially extensible. +(agent-researched, 2026-07-09) + +## Field.Validity render-prop and dynamic aria-describedby registry + +Base UI's `Field.Validity` passes the full `ValidityState` + error array + +current value to a function child, enabling structured error UIs per constraint +violation. `Field.Error` uses a `match` prop (`valueMissing`, `typeMismatch`, +etc.) for conditional rendering. Error elements self-register their ID via +`setMessageIds` and the parent auto-wires `aria-describedby`. + +Multiple descriptions are handled via a `messageIds` array in context — +`FieldDescription` and `FieldError` both append on mount and remove on unmount. +The control's `aria-describedby` is always the joined list of currently-mounted +description IDs. + +**Gap in lingo**: the controller only registers `errorEl` into +`aria-describedby`. When a polite live hint lands (plan 008), that element also +needs `describedby` wiring. Formalizing a `describedByIds` Set with a single +`syncDescribedBy()` call mirrors Base UI's registry and prepares for it. +(agent-researched, 2026-07-09) + +## Validation timing modes and controlled/uncontrolled duality + +Base UI's Field accepts `validationMode: 'onSubmit' | 'onBlur' | 'onChange'` +with `validationDebounceTime`. It tracks `touched`/`dirty`/`valid`/`focused` as +independent booleans (not a string enum). A `useControlled` hook detects +controlled/uncontrolled mode switches and emits dev-mode warnings. + +**Application to lingo**: exposing a `validationTiming` option +(`'eager' | 'blur' | 'submit'`) would map to when `data-invalid`/`aria-invalid` +activate, giving form authors control over error-surfacing rhythm without +forking the controller. The React hook could similarly warn on +controlled/uncontrolled mode switches. (agent-researched, 2026-07-09) + +## Intl.Collator-based filtering for locale-aware completion matching + +Base UI's `useComboboxFilter` uses `Intl.Collator` with `sensitivity:'base'` +for accent-insensitive, case-insensitive prefix matching. This handles +German ss, Turkish dotted-i, and French accents correctly. + +**Gap in lingo**: the completions engine does prefix matching via the registry's +`aliasCompletions` (character-level). For locale packs with accented aliases +(e.g. French "metre" from "metre"), `Intl.Collator`-based comparison would +improve matching without adding dependencies (Intl is allowed per hard rules). +(agent-researched, 2026-07-09) + +## Licenses + +Base UI: MIT (MUI / Radix / Floating-UI team). diff --git a/wiki/research/competitive-landscape.md b/wiki/research/competitive-landscape.md new file mode 100644 index 0000000..1abfa10 --- /dev/null +++ b/wiki/research/competitive-landscape.md @@ -0,0 +1,146 @@ +# Competitive landscape positioning (research 2026-07-09) + +Multi-agent web research pass 2026-07-09. Claims are agent-reported and worth +re-verifying against primary sources before acting on any specific claim. +Subjects: chrono-node v2.9.1, convert-units v2.3.4/v3, js-quantities 1.8.0, +mathjs 15.2.0, UnitMath 1.1.1, Duckling (Facebook, Haskell), Microsoft +Recognizers-Text, compromise, date-fns, Luxon. Cross-referenced with lingo's +existing research in `units-libraries.md` and `date-parsing.md`. + +## Ecosystem fragmentation: nobody unifies the four surfaces + +The NL-parsing and units ecosystem is fragmented along four axes: + +| Capability | Best-in-class today | What it lacks | +|---|---|---| +| NL date parsing | chrono-node (5M weekly, 14 locales) | One-way only; no format or round-trip | +| Unit conversion | convert-units (194K weekly) | Cannot parse text; exact-key input only | +| NL quantity extraction | Duckling (47 languages, 15 dimensions) | Haskell HTTP server; no JS-native; parse-only | +| Duration parsing | ms (Vercel) | No months; silent `undefined` on garbage; one-way | + +No existing library combines parse + convert + validate + format behind spans, a +two-way guarantee, or an LLM tool-boundary layer. Lingo occupies that unique +intersection. + +## What lingo must match (table stakes) + +These capabilities exist in competitors and lingo must be at parity or +consciously narrower: + +1. **chrono-node's known/implied component certainty model.** `isCertain('weekday')` + distinguishes "user said Tuesday" (certain) from "user said 3pm" (weekday + inferred). Lingo's `DateResult` carries grain and alternatives but does not yet + expose per-component certainty (`explicit`|`inferred`|`default`). The gap is + real and matters for the "tools safer" thesis: LLM tool consumers need to know + what the user actually said vs what was assumed. (agent-researched, 2026-07-09) + +2. **chrono-node's strict/casual dual-mode pattern.** `chrono.strict` disables + relative expressions for data processing; `chrono.casual` allows them for + user-facing input. Lingo's date parser could gate on a `mode` option rather + than requiring callers to filter post-parse. (agent-researched, 2026-07-09) + +3. **chrono-node's 14-locale coverage.** Lingo has 7 locale packs today + (en, en-GB, es, fr, pt, zh, ja). The highest-demand gaps are German (de), + Italian (it), Russian (ru), Dutch (nl), and Korean (ko). + (agent-researched, 2026-07-09) + +4. **Duckling's interval/comparative modifier grammar.** Duckling parses + comparative modifiers ("over/above/at least/more than X", "under/below/at + most/less than X") producing structured bound objects. Lingo already handles + ranges, qualifiers, and open-ended bounds, but Duckling's English quantity + test corpus (`Duckling/Quantity/EN/Corpus.hs`) would be a good coverage + validation target. (agent-researched, 2026-07-09) + +5. **convert-units' `toBest()` refinement options.** `cutOffNumber`, `exclude`, + and `system` knobs. Lingo's `pickBestUnit` already has system-aware selection + and a rank ladder; adding `exclude` and `cutOff` options to `ToBestOptions` + matches the proven API surface. (agent-researched, 2026-07-09) + +## What lingo should double down on (exclusive differentiators) + +None of the surveyed competitors have any of these: + +### 1. Spans on every result + +Every lingo parse result carries `{ start, end }` character offsets into the +original input (hard rule 3). chrono-node has `index` (start offset only) but no +end offset and no spans on individual components or issues. convert-units, +js-quantities, mathjs, date-fns, Luxon, and ms have no span concept at all. +Duckling has character offsets but only at the entity level, not on sub-components +or error locations. + +Spans enable: highlighting the problematic range in a form field, pinpointing +where a typo was in the input, and multi-entity extraction with precise source +locations. This is infrastructure that no JS competitor ships. +(agent-researched, 2026-07-09) + +### 2. Two-way guarantee (round-trip invariant) + +`parse(format(x)) === x` is a tested invariant across lingo's entire domain +(hard rule 4). No surveyed competitor guarantees or tests this: + +- chrono-node: parse-only, no format +- convert-units: format returns `{val, unit}` object, does not emit parseable text +- date-fns: `formatDistance` emits qualifiers ("about 1 hour") that don't re-parse +- Luxon: formats/parses only structured ISO/RFC, not natural language +- js-quantities: `toString()` emits canonical unit notation, cannot parse NL +- Duckling: parse-only +- compromise: parse-only + +The guarantee is the strongest possible correctness property for a formatting +library and should be the lead positioning claim. (agent-researched, 2026-07-09) + +### 3. Standard Schema gap: no parser library fills it + +Standard Schema v1 (3.6K GitHub stars) is consumed by Vercel AI SDK, TanStack +Form, react-hook-form resolvers, and shadcn/ui Field. No existing +parsing/units/date library implements it. Zod, Valibot, and ArkType implement +`StandardSchemaV1` for generic validation but perform no domain-specific +canonicalization. Lingo's `./ai` entry implements both `StandardSchemaV1` and +`StandardJSONSchemaV1`, making it the only NL-parsing library that plugs directly +into AI SDK `generateObject()`/`tool()`, TanStack Form validators, and +react-hook-form `standardSchemaResolver` without adapters. +(agent-researched, 2026-07-09) + +### 4. Size wedge + +| Library | Unpacked | min+gzip (est.) | Deps | Scope | +|---|---|---|---|---| +| lingo (full) | — | 36.9 kB | 0 | quantities + dates + durations + ranges + conversion + formatting + fuzzy + completions + locale detection | +| chrono-node | 3.5 MB | ~45 kB | 0 | dates only | +| convert-units | — | ~8 kB | 0 | conversion only (no NL parsing) | +| mathjs (units) | 9.4 MB | ~180 kB | 9 | algebraic units (no NL parsing) | +| Luxon | 4.6 MB | ~23 kB | 0 | date formatting (no NL parsing) | +| compromise | — | ~95 kB | 3 | general NLP | +| date-fns (full) | 10.9 MB | tree-shakeable | 0 | date utility (no NL parsing) | + +Lingo replaces 3-4 partial solutions at a fraction of the combined size. The +marginal cost of each entry point is especially compelling: `lingo/date` adds +~13.1 kB on top of core, vs installing chrono-node separately at ~45 kB. +(agent-researched, 2026-07-09; size numbers from lingo's own `scripts/size.mjs`, +competitor sizes estimated from npm/bundlephobia) + +## Throughput positioning + +Lingo achieves ~493K ops/sec for simple quantities and ~340K ops/sec for dates +on ARM64 (bench baseline). A Duckling HTTP round-trip is orders of magnitude +slower (network + Haskell runtime). chrono-node publishes no throughput +benchmarks beyond trivial cases (empty string, single slash-date). Publishing +comparative numbers on shared corpora would sharpen lingo's performance claim. +(agent-researched, 2026-07-09) + +## Positioning summary + +lingo's competitive moat is the intersection no one else occupies: + +``` +parse NL text + convert + validate + format + round-trip guarantee + + spans on every result + Standard Schema boundary + zero deps + + headless DOM controller + AI tool repair +``` + +The go-to-market angle is *not* "better at one thing" but "the only library that +does all of them together, under 37 kB, with hard correctness properties." The +Standard Schema gap and the two-way guarantee are the most defensible +differentiators because they require architectural commitments competitors cannot +bolt on. diff --git a/wiki/research/date-parsing.md b/wiki/research/date-parsing.md index 581ab33..5beaf2e 100644 --- a/wiki/research/date-parsing.md +++ b/wiki/research/date-parsing.md @@ -46,3 +46,51 @@ Research pass 2026-07-03 (background agent, source-level). Subjects: chrono-node ## Licenses chrono-node MIT · inclusive-dates MIT · Humanizer MIT (.NET Foundation) · date-fns MIT · Luxon MIT · moment MIT · timeago.js MIT · humanize-duration **Unlicense** · ms MIT · tinyduration MIT · Intl spec (ECMA-402). + +## Competitive positioning update (2026-07-09) + +Multi-agent web research pass 2026-07-09, cross-referencing download numbers and +feature matrices across the date-parsing ecosystem. Claims are agent-reported and +worth re-verifying. + +**Download context (npm weekly, agent-reported 2026-07-09)**: chrono-node ~5M, +date-fns ~27M (utility, not NL parsing), Luxon ~11M (utility, not NL parsing), +ms ~130M (minimal durations). + +**chrono-node parity gaps**: + +1. *Known/implied component certainty model* — chrono-node's `isCertain(component)` + is a meaningful distinction lingo does not yet surface. A `certainty` map per + component (year/month/day/hour/minute/tz) with values + `'explicit'|'inferred'|'default'` on `DateResult` would close this. +2. *strict/casual dual-mode* — chrono ships two preconfigured pipelines per + locale. Lingo could expose `mode: 'casual' | 'strict'` where strict disables + deictic expressions and requires two date components. +3. *14-locale coverage* — lingo has 7; the highest-demand missing locales are + de, it, ru, nl, ko. + +**Exclusive lingo differentiators vs chrono-node**: + +- Round-trip: `humanizeDate` output re-parses to the same value (chrono is + parse-only). +- Spans on every date result component (chrono has `index` start only, no end). +- Standard Schema integration (chrono has no schema surface). +- Duration parsing + humanization in the same library (chrono delegates to ms or + date-fns). +- Headless DOM controller for date input fields. +- Size: lingo/date adds ~13.1 kB marginal on top of core; chrono-node is ~45 kB + standalone (both zero deps). +- Throughput: lingo ~340K ops/sec for dates on ARM64 vs unknown for chrono-node + (no published benchmarks beyond trivial cases). + +**Duckling and Recognizers-Text** (server-side, non-JS): + +- Duckling (Facebook, Haskell): 47 languages, 15 dimensions, ranked alternatives, + interval/comparative modifier grammar. Requires deploying a Haskell HTTP server. + No JS-native option. Lingo should validate against Duckling's English test corpus + (`Duckling/Quantity/EN/Corpus.hs`) for coverage parity on modifiers. +- Microsoft Recognizers-Text: 10+ languages, .NET/Java/Python/JS. The JS port + exists but is unmaintained (last publish 2022). Handles "greater than 5 kg" and + "no more than 10 miles" — patterns lingo already parses via qualifier grammar. + +(agent-researched, 2026-07-09; see also `wiki/research/competitive-landscape.md`) diff --git a/wiki/research/library-craft.md b/wiki/research/library-craft.md new file mode 100644 index 0000000..05fbf8e --- /dev/null +++ b/wiki/research/library-craft.md @@ -0,0 +1,183 @@ +# Library-craft lessons: lodash/es-toolkit/remeda, Drizzle ORM, FFmpeg + +Multi-agent web research pass 2026-07-09. Claims are agent-reported and worth +re-verifying against primary sources before acting on any specific claim. +Subjects: lodash (and successors lodash-es, es-toolkit, remeda), Drizzle ORM +v0.45.3, FFmpeg (FATE infrastructure, checkasm, filter graph, deprecation +protocol). Each section distils what those projects do well, what failed, and +what lingo should take. + +## lodash lineage: per-function modularity, size/perf marketing, naming algebra + +**Context**: lodash (60M+ weekly downloads) earned adoption through API naming +consistency (300+ functions with predictable suffix/prefix families: +-By/-With/-In/is-/to-) and one-function-per-page documentation. Its downfall: +pre-ESM packaging made tree-shaking impossible (24 kB gzip for one function), +600+ per-method npm packages fragmented without codemods, and the lodash-es +migration stalled. es-toolkit (Toss, 2024) fixed this with `sideEffects:false`, +per-function files, Vitest `bench()` proving 2-3x perf and up to 97% smaller +bundles function-by-function, plus a compat layer for painless migration. Remeda +added TypeScript-first dual-paradigm APIs with structured JSDoc +(`@signature`/`@example`/`@category`) driving auto-generated docs. + +**Lessons for lingo**: + +1. **Benchmark-driven size/perf marketing per entry point.** es-toolkit ships + `benchmarks/performance/` with per-function head-to-head results published on + the site. Lingo already measures its own throughput (`bench/baseline-node.json`) + and entry-point size (`scripts/size.mjs`). The gap is comparative numbers + against chrono-node, convert-units, and ms on shared inputs, surfaced in docs. + (agent-researched, 2026-07-09) + +2. **Per-function docs pages with runnable examples.** Lodash, es-toolkit, and + Remeda all guarantee every exported function gets its own searchable page. + Lingo already has TSDoc `@example` on every public symbol (143+ verified) but + the site aggregates them on a single long page. Per-function routes for the + top-20 exports would improve discoverability. (agent-researched, 2026-07-09) + +3. **Systematic suffix/prefix naming families.** Lodash: -By/-With/-Deep/-Right. + Lingo already has verb-first families (parse*, humanize*, convert*, register*, + define*, is*) documented in `wiki/conventions.md`, plus established suffixes + -Delta (convertDelta) and -Range (parseRange). Formalizing this as a naming + algebra in `wiki/api-design.md` prevents ad hoc names on future additions. + (agent-researched, 2026-07-09) + +4. **Per-entry size badges in published docs.** es-toolkit measures each + function individually with esbuild min+gzip and publishes the numbers. Lingo + already measures identically via `scripts/size.mjs`; the data just needs to be + piped to a JSON file the docs site renders. (agent-researched, 2026-07-09) + +5. **Compat-layer migration on-ramp from incumbents.** es-toolkit ships + `es-toolkit/compat` accepting lodash's signature. A future + `@pascal-app/lingo/compat/chrono` accepting chrono-node's `parse()` signature + would lower the adoption barrier. Even migration recipes in docs (without code) + would capture intent. (agent-researched, 2026-07-09) + +6. **Inline JSDoc as the single source for docs generation.** Remeda generates + its site from `@signature`/`@example`/`@category`. Lingo could add `@category` + and `@since` tags, then wire `scripts/gen-docs-catalog.mjs` to emit the + site's nav structure mechanically. (agent-researched, 2026-07-09) + +## Drizzle ORM: zero-dep schema-as-types, multi-library bridge, dev/runtime split + +**Context**: Drizzle ORM (v0.45.3, 35.1k stars) is a zero-dependency TypeScript +ORM (~7.4 kB min+gzip) whose core insight is that schema declarations *are* the +type system. Column builders progressively narrow types via branded intersection +helpers (`NotNull`, `HasDefault`); `$inferSelect`/`$inferInsert` derive +exact model types from accumulated builder state with zero runtime cost. Dev +tooling (`drizzle-kit`) never enters the production bundle. Multi-validation +bridges (`drizzle-zod`, `drizzle-valibot`, `drizzle-typebox`, `drizzle-arktype`) +emit target-library schemas from a single source. + +**Lessons for lingo**: + +1. **`$inferInput`/`$inferOutput` type accessors on LingoField.** Drizzle + exposes `typeof table.$inferSelect` and `$inferInsert` from one schema. Lingo + fields could expose `typeof field.$inferOutput` (canonical value: number or + QuantityJSON) and `$inferInput` (string — what models emit). Purely + type-level, zero runtime cost. Directly serves plan 027's literal-typed + ecosystem. (agent-researched, 2026-07-09) + +2. **Branded intersection helpers for progressive type narrowing.** Drizzle + chains `.notNull()`, `.default()`, `.$type()` where each returns a builder + with an additional branded property. Lingo's `quantityField` currently uses + flat option overloads; a builder pattern could refine the Output type parameter + progressively. (agent-researched, 2026-07-09) + +3. **Multi-validation-library bridge from a single schema.** `drizzle-zod` etc. + iterate columns and emit target-library schemas. Lingo's `lingoObject` could + adopt the conditions-pattern to emit Zod/Valibot schemas for consumers who + need them — as optional peer-dep adapters, not new runtime deps. + (agent-researched, 2026-07-09) + +4. **Dev-tooling separation: heavy introspection never enters prod.** Drizzle's + `drizzle-kit` is devDeps-only. Lingo's JSON Schema generation functions + (`quantityJsonSchema`, `quantityRangeJsonSchema`) are only needed at + tool-definition time. Marking them `/* @__PURE__ */` would help bundlers + eliminate them when only `.parse()` is used at runtime. + (agent-researched, 2026-07-09) + +5. **`const`-generic identity helpers for literal preservation.** Drizzle's + `pgTable('name', { columns })` preserves literal column names via ``. + Lingo already uses this in `defineKind`; extending it to `lingoObject`'s shape + parameter (`(shape: S)`) would give exact + key unions on `InferLingoObject`. (agent-researched, 2026-07-09) + +6. **`(string & {})` escape hatch for literal types with dynamic fallback.** + Lingo already applies this on `Kind`. The audit notes that + `QuantityFieldOptions.unit` is typed as plain `string` — it should be + `UnitRefByKind | (string & {})` for autocomplete on known units while + accepting custom registry units. (agent-researched, 2026-07-09) + +## FFmpeg: FATE-level regression discipline, fuzz/differential testing, pipeline formalism + +**Context**: FFmpeg maintains 5,500+ automated FATE tests with 4,800+ reference +fixture files enforcing exact-output regression detection across 100+ platform/ +compiler configurations. `checkasm` provides per-function cycle-level +micro-benchmarking. The filter graph is a composable pipeline of uniform +`AVFilter` interfaces. A strict versioned deprecation policy (`FF_API_*` guards, +major-bump-only removal) keeps the CLI/library surface coherent. + +**Lessons for lingo**: + +1. **CI-blocking corpus regression with additive/breaking classification.** + FATE blocks on any reference mismatch; regeneration is explicit (`GEN=1`). + Lingo's `corpus-diff.mjs` already classifies ADDITIVE vs BREAKING. The gap is + growing corpus breadth (currently ~300 entries; FFmpeg covers every codec) and + adding it as a blocking CI step. (agent-researched, 2026-07-09) + +2. **Fuzz testing the parser with coverage-guided inputs.** FFmpeg integrates + with OSS-Fuzz (3 sanitizers, 120+ targets). Lingo's `hostile.test.ts` covers + 38 hand-written adversarial cases; a lightweight fuzz harness (random/mutated + UTF-16 strings, asserting never-throw + finite-values + linear-time) on + nightly CI would catch normalizer/tokenizer edge cases. Aligns with plan 010 + layer 2. (agent-researched, 2026-07-09) + +3. **Differential testing: parse(format(x)) round-trip oracle.** FATE's + enc_dec_pcm pattern encodes then decodes and compares. Lingo's two-way + guarantee (hard rule 4) is tested per-case but not systematically exhausted. + A property-based oracle — for every bench corpus value call `format()` then + `parse()` and assert `base` equality within epsilon — catches format/parse + desync automatically. This is the seeded property round-trip test plan 010 + spec'd but has not yet shipped. (agent-researched, 2026-07-09) + +4. **Benchmark baseline comparison in CI with regression threshold.** FFmpeg's + `checkasm --compare` detects perf regressions above a threshold. Lingo's + `bench.mjs` already has `--write-baseline` and `--compare --threshold 30`. + The gap is checking in a `bench/baseline-node.json` on a stable runner and + making it a CI gate. (agent-researched, 2026-07-09) + +5. **Hostile-input latency probes as CI gates.** FFmpeg's `checkasm` includes + adversarial probes with resource limits. Lingo already has 3 latency probes + (50k no-match, 20k unknown tail, 500-digit number). Promoting them to CI + gates with wall-clock budgets prevents catastrophic backtracking from + shipping. (agent-researched, 2026-07-09) + +6. **Versioned deprecation guards for public API changes.** FFmpeg uses + `FF_API_*` preprocessor guards tied to major versions. As lingo approaches + 1.0, adopting `@deprecated` JSDoc + a `LINGO_DEPRECATED_*` constant that + emits a one-time dev-only `console.warn` (tree-shaken in prod) would give + users clear migration windows without runtime cost. (agent-researched, + 2026-07-09) + +7. **Formalize pipeline stages as a uniform interface.** FFmpeg's `AVFilter` + contract (named pads, format negotiation, frame-based activation) allows + arbitrary composition. Lingo's data flow (normalize, tokenize, grammar, + result, format) already exists as stages but is not individually addressable + by callers. Extracting a typed `Stage` interface would enable custom + grammar rules, middleware, and per-stage testing. Large effort; deferred. + (agent-researched, 2026-07-09) + +## Cross-cutting themes + +- **Zero runtime deps is validated.** Drizzle (~7.4 kB, 0 deps) and es-toolkit + prove the strategy works at scale. Both projects make it a marketing point. +- **Size gates as a competitive wedge.** es-toolkit publishes per-function sizes; + Drizzle separates dev tooling. Both validate lingo's `scripts/size.mjs` gate + and entry-point architecture. +- **Property tests and round-trip oracles beat snapshot maintenance.** FFmpeg's + enc_dec pattern and plan 010's spec converge on the same idea; lingo should + ship it. +- **Naming predictability reduces docs burden.** Lodash's suffix families let + devs predict API names before looking them up; lingo's verb-first families are + already there, but documenting the algebra explicitly makes it self-reinforcing. diff --git a/wiki/research/units-libraries.md b/wiki/research/units-libraries.md index d998979..07255c4 100644 --- a/wiki/research/units-libraries.md +++ b/wiki/research/units-libraries.md @@ -69,3 +69,35 @@ Factors are uncopyrightable facts; attribute alias lists borrowed from Apache-2. **Steal**: libphonenumber DX shape (rich value object, lenient extract + strict mode, isPossible/isValid tiers, reason codes, findInText offsets, AsYouType, metadata tiers); js-quantities alias craft + temp delta split; UnitMath config discipline + formatPrefixes; mathjs splitUnit + exact-rational habit; convert-units toBest knobs + describe(); Intl as display backend. **Avoid**: rounded anchors; silent binary/decimal switches; context-free single-letter aliases; case-insensitive symbol matching; alias collisions without build-time rejection; arithmetic on absolute temp scales; mixed regional conventions in one table; undocumented calendar averages; mutable global registries. None of the six libs parse `5'11"`, ′/″/℃ codepoints, `1½ cups`, decimal commas, spaced `sq ft`, `cbm`, troy oz, or ranges — that surface is ours. + +## Competitive positioning update (2026-07-09) + +Multi-agent web research pass 2026-07-09, cross-referencing download numbers and +feature matrices across the units/measurement ecosystem. Claims are +agent-reported and worth re-verifying. + +**Download context (npm weekly, agent-reported 2026-07-09)**: convert-units ~194K, +js-quantities (not separately tracked, bundled in projects), mathjs ~1.6M (full +math suite, not just units). None parses natural-language text; all require exact +programmatic keys. + +**Exclusive lingo differentiators vs the units ecosystem**: + +1. Spans on every result — no units library ships character offsets. +2. Two-way guarantee — none guarantees `parse(format(x)) === x`. +3. Standard Schema implementation — no units library is a Standard Schema + validator, so none plugs into AI SDK, TanStack Form, or react-hook-form + without adapter code. +4. Size: lingo full (36.9 kB gz, 0 deps) is comparable to convert-units alone + (~8 kB gz, conversion only) while covering parsing + conversion + formatting + + fuzzy + completions. +5. NL parsing of compound forms (`5'11"`, `1½ cups`, spaced `sq ft`, decimal + commas, primes/smart quotes) — surface no competitor handles. + +**Gaps to close**: convert-units' `toBest()` `exclude`/`cutOff` options are not +yet matched in lingo's `pickBestUnit`; js-quantities' troy-oz coverage shows a +real-world demand lingo now serves (D-entry exists). The `(string & {})` escape +hatch on Kind should extend to field options (`QuantityFieldOptions.unit`) for +autocomplete-with-fallback. + +(agent-researched, 2026-07-09; see also `wiki/research/competitive-landscape.md`) From 4bd595151a19d5716c6ef887e1fba486d95b8b1d Mon Sep 17 00:00:00 2001 From: Aymeric Rabot Date: Thu, 9 Jul 2026 12:00:16 +0200 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20locale=20idiom=20coverage=20wave=20?= =?UTF-8?q?1=20=E2=80=94=20CJK=20numbers,=20Romance=20composition,=20local?= =?UTF-8?q?ized=20dates=20(D68/D69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Research pass across es/fr/pt/zh/ja/en (335 test-precise gap idioms, wiki/research/locale-idioms.md, plan 033) implemented as language-neutral engine seams consumed through new optional pack-data fields — no per-language parser branches (the chrono-node anti-pattern). Engine (D68) - CJK number walker: 三公斤/三十五キロ sub-token segmentation, 万/亿 grouping (三百五十万), elliptical 一百五=150 / 三万五, mixed 3万5千, wave-dash ranges 5〜10, adjacent-number ranges 七八天, post-unit 半; 万万/亿万 rejected; D47 glued-1M guard holds. - Romance composition: treinta y cinco, bare scales (cien gramos), composed table (quinientos, exhaustive vigesimal quatre-vingt-dix), spoken decimals as digit sequences (dos coma cinco seis = 2.56; English two point five), article-led chains (un millón quinientos mil), multi-word approximants (más o menos) with longest-first matching. Dates (D69) - Pack-owned spoken clock (las tres menos cuarto, quinze para as tres, quarter of five), period edges (fin juillet, 月底), weekday offsets (lundi en huit, Monday week), afterNext/beforeLast (再来週, 先々週), day-part compounds (tomorrow morning), localized durations (2 horas). English clock grammar migrated to the same fields, behavior-identical. Packs + gates - es/fr/pt/zh/ja deepened (~200 locale corpus rows); per-locale corpus contracts discovered generically and enforced in bun run check; locale auto-detection scores the new number tables. Traditional units 斤/两 deliberately not registered (D4 hazards, backlog). - Budgets recalibrated once for the wave (D68/D69 entries): full 38.3, core 25.6, date 41.0/14.4, ai 17.4, pack budgets raised for restored breadth. Main corpus: zero interpretation changes (additive only). Cross-reviewed adversarially (Codex ↔ Opus); both review blockers fixed (digit-sequence decimals, article-led chains) plus 万万 rejection, detection scoring, veintiuna, longest-first phrase contract. Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Codex GPT-5.5 --- packages/lingo/CHANGELOG.md | 31 + packages/lingo/scripts/corpus-diff.mjs | 125 +- packages/lingo/scripts/size.mjs | 73 +- packages/lingo/src/date/date.test.ts | 311 ++++- packages/lingo/src/date/duration.ts | 26 + packages/lingo/src/date/parse.ts | 2 + packages/lingo/src/date/relative.ts | 246 +++- packages/lingo/src/date/time.ts | 250 +++- packages/lingo/src/date/vocab.ts | 4 +- packages/lingo/src/locale/detect.ts | 17 +- packages/lingo/src/locale/en-core.ts | 2 + packages/lingo/src/locale/profile.ts | 267 ++-- packages/lingo/src/locale/types.ts | 50 +- packages/lingo/src/locales/es.ts | 54 +- packages/lingo/src/locales/fr.ts | 59 + packages/lingo/src/locales/ja.ts | 23 + packages/lingo/src/locales/pt.ts | 44 +- packages/lingo/src/locales/zh.ts | 30 +- packages/lingo/src/number/cjk.test.ts | 47 + packages/lingo/src/number/cjk.ts | 246 ++++ .../lingo/src/number/words-romance.test.ts | 622 ++++++++++ packages/lingo/src/number/words.ts | 202 ++- packages/lingo/src/parse/normalize.ts | 3 + packages/lingo/src/parse/quantity.ts | 12 + packages/lingo/src/parse/range.ts | 75 +- packages/lingo/tests/corpus/contract-v1.json | 26 + .../tests/corpus/locale-contract.test.mjs | 32 + .../tests/corpus/locale-es-contract-v1.json | 946 ++++++++++++++ .../lingo/tests/corpus/locale-es-source.mjs | 233 ++++ .../tests/corpus/locale-fr-contract-v1.json | 945 ++++++++++++++ .../lingo/tests/corpus/locale-fr-source.mjs | 233 ++++ .../tests/corpus/locale-ja-contract-v1.json | 690 +++++++++++ .../lingo/tests/corpus/locale-ja-source.mjs | 200 +++ .../tests/corpus/locale-pt-contract-v1.json | 1102 +++++++++++++++++ .../lingo/tests/corpus/locale-pt-source.mjs | 241 ++++ .../tests/corpus/locale-zh-contract-v1.json | 599 +++++++++ .../lingo/tests/corpus/locale-zh-source.mjs | 196 +++ packages/lingo/tests/corpus/source.mjs | 2 + packages/lingo/tests/locales/cjk.test.ts | 273 ++++ packages/lingo/tests/locales/romance.test.ts | 375 +++++- plans/033-locale-idiom-coverage.md | 82 ++ plans/backlog.md | 60 + wiki/decisions.md | 58 + wiki/inspiration.md | 9 + wiki/research/locale-idioms.md | 950 ++++++++++++++ 45 files changed, 9754 insertions(+), 319 deletions(-) create mode 100644 packages/lingo/src/number/cjk.test.ts create mode 100644 packages/lingo/src/number/cjk.ts create mode 100644 packages/lingo/src/number/words-romance.test.ts create mode 100644 packages/lingo/tests/corpus/locale-contract.test.mjs create mode 100644 packages/lingo/tests/corpus/locale-es-contract-v1.json create mode 100644 packages/lingo/tests/corpus/locale-es-source.mjs create mode 100644 packages/lingo/tests/corpus/locale-fr-contract-v1.json create mode 100644 packages/lingo/tests/corpus/locale-fr-source.mjs create mode 100644 packages/lingo/tests/corpus/locale-ja-contract-v1.json create mode 100644 packages/lingo/tests/corpus/locale-ja-source.mjs create mode 100644 packages/lingo/tests/corpus/locale-pt-contract-v1.json create mode 100644 packages/lingo/tests/corpus/locale-pt-source.mjs create mode 100644 packages/lingo/tests/corpus/locale-zh-contract-v1.json create mode 100644 packages/lingo/tests/corpus/locale-zh-source.mjs create mode 100644 plans/033-locale-idiom-coverage.md create mode 100644 wiki/research/locale-idioms.md diff --git a/packages/lingo/CHANGELOG.md b/packages/lingo/CHANGELOG.md index 266ca19..8396523 100644 --- a/packages/lingo/CHANGELOG.md +++ b/packages/lingo/CHANGELOG.md @@ -9,6 +9,37 @@ change**, even if the API is untouched. ### Added +- **Locale idiom coverage wave 1** (plan 033, D68/D69): parse how people + actually write and speak quantities and dates across all shipped locales. + - CJK number engine: multi-character number words inside CJK tokens + (`三公斤`, `三十五キロ`), 万/亿/億 scale grouping (`三百五十万` = 3.5M), + elliptical shorthands (`一百五` = 150, `三万五` = 35 000), mixed + digit+scale (`3万5千`, `1億2千万`), wave-dash ranges (`5〜10キロ`), + adjacent-number ranges (`七八天` = 7–8 days), and post-unit 半 + (`两公斤半` = 2.5 kg). + - Romance number composition via new pack fields: tens + and-word + ones + (`treinta y cinco`), `bareScales` (`cien gramos`, `mil metros`), + `composed` exact compounds (`quinientos`, exhaustive French vigesimal + `quatre-vingt-dix`), and spoken decimals via `decimalWords` + (`dos coma cinco`, `trois virgule quatorze`; English gains + `two point five` through the same table). + - Localized date grammar via new pack fields: spoken clock + (`las tres menos cuarto`, `quinze para as tres`, `deux heures et quart`, + US `quarter of five`), period edges (`fin juillet`, `a finales de mes`, + `月底`), weekday offsets (`lundi en huit`, `Monday week`, + `Tuesday fortnight`), after-next/before-last modifiers (`再来週`, + `先々週`, `the week after next`), day + day-part compounds + (`tomorrow morning`), and duration parsing that honors pack unit words + (`2 horas`). + - Multi-word leading approximants (`más o menos`, `à peu près`, + `por volta de`) and trailing approximants (`y pico`, `e pouco`). + - Deepened es/fr/pt/zh/ja packs (day offsets like `pasado mañana`, + `avant-hier`, `前天`, `一昨日`; fuzzy amounts `une vingtaine`; + duration words `个小时`, `時間`; `円` → JPY) — ~190 new locale corpus + rows. + - Per-locale corpus gates: `tests/corpus/locale--source.mjs` → + checked-in contracts, discovered generically and enforced by + `bun run check`, so locale behavior can no longer silently regress. - `isNumber()` result guard, completing the `isQuantity`/`isRange`/`isConversion` family for the bare-number branch of `LingoResult`. - DOM completion fields now ship the headless half of the WAI-ARIA combobox diff --git a/packages/lingo/scripts/corpus-diff.mjs b/packages/lingo/scripts/corpus-diff.mjs index 8a9f9ed..5da593d 100644 --- a/packages/lingo/scripts/corpus-diff.mjs +++ b/packages/lingo/scripts/corpus-diff.mjs @@ -1,4 +1,4 @@ -import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' import { buildContract } from '../tests/corpus/source.mjs' const ROOT = new URL('..', import.meta.url) @@ -15,7 +15,7 @@ if (!(existsSync(indexUrl) && existsSync(dateUrl))) { process.exit(1) } -const [{ lingo }, { parseDate, parseDateRange }] = await Promise.all([ +const [{ lingo, createLingo }, { parseDate, parseDateRange }] = await Promise.all([ import(indexUrl.href), import(dateUrl.href), ]) @@ -25,33 +25,126 @@ const current = buildContract({ lingo, parseDate, parseDateRange }) if (write) { writeFileSync(CONTRACT_URL, `${JSON.stringify(current, null, 2)}\n`) console.log(`Wrote ${relative(CONTRACT_URL)}`) - process.exit(0) } -if (!existsSync(CONTRACT_URL)) { +if (!(write || existsSync(CONTRACT_URL))) { console.error( `${relative(CONTRACT_URL)} is missing. Run node scripts/corpus-diff.mjs --write after build.`, ) process.exit(reportOnly ? 0 : 1) } -const expected = JSON.parse(readFileSync(CONTRACT_URL, 'utf8')) -const changes = diffContract(expected, current) -const additive = changes.filter((change) => change.classification === 'ADDITIVE') -const breaking = changes.filter((change) => change.classification === 'BREAKING') - -if (changes.length === 0) { - console.log('Corpus contract: zero changes.') -} else { - console.log(`Corpus contract: ${additive.length} additive, ${breaking.length} breaking.`) - for (const change of changes) { - console.log(`${change.classification} ${change.path}: ${change.reason}`) +if (!write) { + const expected = JSON.parse(readFileSync(CONTRACT_URL, 'utf8')) + const changes = diffContract(expected, current) + const additive = changes.filter((change) => change.classification === 'ADDITIVE') + const breaking = changes.filter((change) => change.classification === 'BREAKING') + + if (changes.length === 0) { + console.log('Corpus contract: zero changes.') + } else { + console.log(`Corpus contract: ${additive.length} additive, ${breaking.length} breaking.`) + for (const change of changes) { + console.log(`${change.classification} ${change.path}: ${change.reason}`) + } + } + + if (breaking.length > 0 && !reportOnly) { + process.exit(1) + } +} + +// ─── Locale corpus discovery ──────────────────────────────────────────────── +// Auto-discover locale-*-source.mjs files and check/write their contracts. + +const corpusDir = new URL('tests/corpus/', ROOT) +const localeSourceFiles = readdirSync(corpusDir) + .filter((f) => /^locale-[a-z]+-source\.mjs$/.test(f)) + .sort() + +let localePacksCache = null +async function getLocalePacks() { + if (localePacksCache) { + return localePacksCache + } + const localeDir = new URL('dist/locales/', ROOT) + if (!existsSync(localeDir)) { + return [] + } + const files = readdirSync(localeDir).filter((f) => f.endsWith('.js') && f !== 'index.js') + const packs = [] + for (const f of files) { + const mod = await import(new URL(`dist/locales/${f}`, ROOT).href) + const pack = mod.default ?? Object.values(mod)[0] + if (pack?.locale) { + packs.push(pack) + } + } + localePacksCache = packs + return packs +} + +let localeBreaking = false +for (const sourceFile of localeSourceFiles) { + const id = sourceFile.replace('locale-', '').replace('-source.mjs', '') + const contractFile = `locale-${id}-contract-v1.json` + const contractUrl = new URL(`tests/corpus/${contractFile}`, ROOT) + const sourceUrl = new URL(`tests/corpus/${sourceFile}`, ROOT) + + const localeMod = await import(sourceUrl.href) + const localePacks = await getLocalePacks() + const localeInstance = createLingo({ locales: localePacks }) + const localeLingo = localeInstance.parse + const localeCurrent = localeMod.buildContract({ + lingo: localeLingo, + localePacks, + parseDate, + parseDateRange, + }) + + if (write) { + writeFileSync(contractUrl, `${JSON.stringify(localeCurrent, null, 2)}\n`) + console.log(`Wrote ${relative(contractUrl)}`) + continue + } + + if (!existsSync(contractUrl)) { + console.error( + `${relative(contractUrl)} is missing. Run node scripts/corpus-diff.mjs --write after build.`, + ) + if (!reportOnly) { + localeBreaking = true + } + continue + } + + const localeExpected = JSON.parse(readFileSync(contractUrl, 'utf8')) + const localeChanges = diffContract(localeExpected, localeCurrent) + const localeAdditive = localeChanges.filter((c) => c.classification === 'ADDITIVE') + const localeBreakingChanges = localeChanges.filter((c) => c.classification === 'BREAKING') + + if (localeChanges.length === 0) { + console.log(`Locale corpus [${id}]: zero changes.`) + } else { + console.log( + `Locale corpus [${id}]: ${localeAdditive.length} additive, ${localeBreakingChanges.length} breaking.`, + ) + for (const change of localeChanges) { + console.log(` ${change.classification} ${change.path}: ${change.reason}`) + } + } + + if (localeBreakingChanges.length > 0 && !reportOnly) { + localeBreaking = true } } -if (breaking.length > 0 && !reportOnly) { +if (localeBreaking) { process.exit(1) } +if (write) { + process.exit(0) +} function diffContract(expectedRoot, currentRoot) { const changes = [] diff --git a/packages/lingo/scripts/size.mjs b/packages/lingo/scripts/size.mjs index d21c8c5..1c1d668 100644 --- a/packages/lingo/scripts/size.mjs +++ b/packages/lingo/scripts/size.mjs @@ -138,37 +138,72 @@ function check(label, size, budget) { // 36.9 (was 36.4): D66 — locale date vocab tables/frames are pack-owned // (deictics, day parts, relative offset markers, period words, compact CJK // offsets) and the shared profile merger/detector knows those optional fields. +// 38.3 (was 36.9): D68 — wave-1 idiom engine: Romance number composition +// (tens+and+ones, bareScales, composed table incl. vigesimal, decimalWords, +// approximatePhrases) + CJK number engine (sub-token segmentation, 万/亿 +// grouping, elliptical/mixed forms, wave-dash + adjacent ranges, post-unit 半). +// Measured 38.19 after golfing; capability is product (D14 pattern). const full = await bundleStdin(`export * from './src/index.ts'`) -check('lingo (full)', full, 36_900) +check('lingo (full)', full, 38_300) if (has('src/locales/es.ts')) { const enLocale = await bundleStdin(`export * from './src/locales/en.ts'`) check('./locales/en (standalone data)', enLocale, 2100) + // 1.9 (was 1.8): D69 — plan 033 locale idiom coverage: composed table + // (veintiuno-veintinueve fused forms + doscientos-novecientos compound + // hundreds), bareScales, decimalWords, approximatePhrases, trailingApproxPhrases, + // rangeAlternativeWords, rangeFromWords 'de', dayOffsets, fuzzyAmounts, approxWords. + // (raised again for restored breadth entries — coverage beats pack-byte golf, D14) const esLocale = await bundleStdin(`export * from './src/locales/es.ts'`) - check('./locales/es (standalone data)', esLocale, 1600) + check('./locales/es (standalone data)', esLocale, 1900) + // 2.1 (was 1.9): D69 — plan 033: exhaustive vigesimal composed table + // (soixante-dix..quatre-vingt-dix-neuf per CLDR RBNF), bareScales, + // decimalWords, approximatePhrases, trailingApproxPhrases/Words, + // rangeAlternativeWords, fuzzyAmounts (dizaine/vingtaine/centaine/millier), + // dayOffsets, dayTimePhrases. + // (raised again for restored breadth entries — coverage beats pack-byte golf, D14) const frLocale = await bundleStdin(`export * from './src/locales/fr.ts'`) - check('./locales/fr (standalone data)', frLocale, 1600) + check('./locales/fr (standalone data)', frLocale, 2100) + // 1.9 (was 1.7): D69 — plan 033: composed table (duzentos-novecentos + // compound hundreds + PT/BR regional teen variants), bareScales, decimalWords, + // approximatePhrases, trailingApproxPhrases, rangeAlternativeWords, + // dayOffsets, fuzzyAmounts, approxWords. + // (raised again for restored breadth entries — coverage beats pack-byte golf, D14) const ptLocale = await bundleStdin(`export * from './src/locales/pt.ts'`) - check('./locales/pt (standalone data)', ptLocale, 1600) + check('./locales/pt (standalone data)', ptLocale, 1900) + // 1.3 (was 1.2): D69 — plan 033: CJK number tables for the D68 walker + // (两/幺/点/半, scales), duration unit aliases (个小时/時間), trailing approx + // (左右/上下), day offsets (前天/一昨日), period edges, 円/JPY. const zhLocale = await bundleStdin(`export * from './src/locales/zh.ts'`) - check('./locales/zh (standalone data)', zhLocale, 1200) + check('./locales/zh (standalone data)', zhLocale, 1300) + // 1.4 (was 1.2): D69 — see zh note; ja adds 再来週/先々週 modifiers and + // 今朝/今晩 day-part phrases. const jaLocale = await bundleStdin(`export * from './src/locales/ja.ts'`) - check('./locales/ja (standalone data)', jaLocale, 1200) + check('./locales/ja (standalone data)', jaLocale, 1400) const enGbLocale = await bundleStdin(`export * from './src/locales/en-gb.ts'`) check('./locales/en-gb (standalone data)', enGbLocale, 250) const withRomanceLocales = await bundleStdin( `export * from './src/index.ts'; export { es } from './src/locales/es.ts'; export { fr } from './src/locales/fr.ts'; export { pt } from './src/locales/pt.ts'`, ) - check('./locales es+fr+pt (marginal over full)', withRomanceLocales - full, 3100) + // 4.3 (was 3.9): D69 — plan 033 romance idiom composed tables; composed + // vigesimal (FR, 30 entries) + compound hundreds (ES 28, PT 16) + bareScales + + // decimal/approx/range/date fields across all three packs. Low cross-locale gzip + // deduplication because each language has unique words. + // (raised again for restored breadth entries — coverage beats pack-byte golf, D14) + check('./locales es+fr+pt (marginal over full)', withRomanceLocales - full, 4300) const withCjkLocales = await bundleStdin( `export * from './src/index.ts'; export { zh } from './src/locales/zh.ts'; export { ja } from './src/locales/ja.ts'`, ) - check('./locales zh+ja (marginal over full)', withCjkLocales - full, 1600) + // 2.0 (was 1.6): D69 — plan 033 CJK idiom pack data (see zh/ja standalone + // notes); low zh/ja gzip dedup because the scripts differ. + check('./locales zh+ja (marginal over full)', withCjkLocales - full, 2000) const withAllLocales = await bundleStdin( `export * from './src/index.ts'; export { es } from './src/locales/es.ts'; export { fr } from './src/locales/fr.ts'; export { pt } from './src/locales/pt.ts'; export { zh } from './src/locales/zh.ts'; export { ja } from './src/locales/ja.ts'; export { enGb } from './src/locales/en-gb.ts'`, ) - check('./locales all loaded (marginal over full)', withAllLocales - full, 4500) + // 6.3 (was 5.8): D69 — plan 033 romance idiom packs (see above). + // (raised again for restored breadth entries — coverage beats pack-byte golf, D14) + check('./locales all loaded (marginal over full)', withAllLocales - full, 6300) } // 19.9 (was 19.6): D48 — shared parser recognizes GBP pence idioms and @@ -210,8 +245,11 @@ if (has('src/locales/es.ts')) { // fails. Keeps auto mode robust without importing locale data into core. // 24.2 (was 23.8): D66 — the shared locale profile merge/detection layer knows // the optional date-vocab fields; the actual language strings remain in packs. +// 25.6 (was 24.2): D68 — wave-1 idiom engine mechanisms live in the shared +// number/parse layer (Romance composition fields + CJK number walker), so +// BYO-registry cores get them too. Measured 25.46 after golfing. const core = await bundleStdin(`export * from './src/core/index.ts'`) -check('./core (engine, no unit data)', core, 24_200) +check('./core (engine, no unit data)', core, 25_600) if (has('src/date/index.ts')) { const dateAlone = await bundleStdin(`export * from './src/date/index.ts'`) @@ -259,7 +297,12 @@ if (has('src/date/index.ts')) { // 38.3 (was 38.2): D67 — anchored-range two-way fix (time-grain "N hours // starting " humanize phrasing) + range-zone threading through the // anchored path (measured 38.26 after golfing; correctness is product, D14). - check('./date (standalone, incl. engine)', dateAlone, 38_300) + // 39.7 (was 38.3): D68 — standalone date inherits the wave-1 idiom engine + // through the shared number/parse layer (measured 39.51 after golfing). + // 41.0 (was 39.7): D69 — localized date grammar seams (clock past/to/minute + // words, period edges, weekday offsets, afterNext/beforeLast, day-part + // compounds, localized durations). Measured 40.85 after golfing. + check('./date (standalone, incl. engine)', dateAlone, 41_000) const withDate = await bundleStdin( `export * from './src/index.ts'; export * from './src/date/index.ts'`, ) @@ -279,7 +322,9 @@ if (has('src/date/index.ts')) { // 12.9 (was 12.6): D64 — locale unit-vocab detection cascade (see the // standalone note above). // 13.1 (was 12.9): D66 — date-only parser readers for locale date vocab. - check('./date (marginal over full)', withDate - full, 13_100) + // 14.4 (was 13.1): D69 — localized date grammar consumers live entirely in + // the date module (see standalone note). Measured 14.29 after golfing. + check('./date (marginal over full)', withDate - full, 14_400) } if (has('src/dom/index.ts')) { @@ -398,7 +443,9 @@ if (has('src/ai/index.ts')) { // machinery still tree-shakes out of /ai. The D64 detector grows `full`, so it // cancels in this marginal — the growth here is date-grammar weight only. // 16.1 (was 15.7): D66 — /ai date fields bundle the locale-aware date parser. - check('./ai (marginal over full)', withAi - full, 16_100) // D30: +notation in shared renderNumber + // 17.4 (was 16.1): D69 — the localized date grammar cascades through the + // bundled date module (see ./date notes). Measured 17.27 after golfing. + check('./ai (marginal over full)', withAi - full, 17_400) // D30: +notation in shared renderNumber if (has('src/mcp/index.ts')) { const withMcp = await bundleStdin( `export * from './src/index.ts'; export * from './src/ai/index.ts'; export * from './src/mcp/index.ts'`, diff --git a/packages/lingo/src/date/date.test.ts b/packages/lingo/src/date/date.test.ts index 9eef653..1d11657 100644 --- a/packages/lingo/src/date/date.test.ts +++ b/packages/lingo/src/date/date.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' -import { humanizeDuration, parseDate, parseDuration } from './index' +import type { LocalePack } from '../locale/types' +import { humanizeDate, humanizeDuration, parseDate, parseDuration } from './index' const NOW = new Date(2026, 6, 3, 14, 30, 0) const MONTH_NAMES = [ @@ -36,8 +37,12 @@ function expectOkDate( return result } -function expectDuration(input: string, seconds: number) { - const result = parseDuration(input) +function expectDuration( + input: string, + seconds: number, + opts: Parameters[1] = {}, +) { + const result = parseDuration(input, opts) if (!result.ok) { throw new Error(`parseDuration("${input}") failed: ${JSON.stringify(result.issues)}`) } @@ -258,6 +263,205 @@ describe('parseDate weekdays and calendar periods', () => { expectDateRoundTrip(last, julyNow) expectDateRoundTrip(next, julyNow) }) + + it('parses day-part compounds from day offsets and weekdays', () => { + expectOkDate('tomorrow morning', new Date(2026, 6, 4, 9), 'hour') + expectOkDate('yesterday evening', new Date(2026, 6, 2, 19), 'hour') + expectOkDate('Friday morning', new Date(2026, 6, 3, 9), 'hour', { + now: new Date(2026, 6, 2, 12), + }) + + const noNow = parseDate('tomorrow morning') + expect(noNow.ok).toBe(false) + expect(!noNow.ok && noNow.issues.some((issue) => issue.code === 'NOW_REQUIRED')).toBe(true) + }) + + it('keeps English humanize output parseable for newly accepted English idioms', () => { + for (const input of ['tomorrow morning', 'Monday week']) { + const parsed = parseDate(input, { now: NOW }) + expect(parsed.ok, input).toBe(true) + if (!parsed.ok) { + continue + } + const phrase = humanizeDate(parsed.date, { now: NOW }) + const reparsed = parseDate(phrase, { now: NOW }) + expect(reparsed.ok, `${input} -> ${phrase}`).toBe(true) + } + }) + + it('parses weekday-offset phrases from pack data', () => { + expectOkDate('Monday week', new Date(2026, 6, 13), 'day') + expectOkDate('a week Monday', new Date(2026, 6, 13), 'day') + expectOkDate('week on Tuesday', new Date(2026, 6, 14), 'day') + expectOkDate('Tuesday fortnight', new Date(2026, 6, 21), 'day') + const noNow = parseDate('Monday week') + expect(noNow.ok).toBe(false) + expect(!noNow.ok && noNow.issues.some((issue) => issue.code === 'NOW_REQUIRED')).toBe(true) + + const frWeekdayOffset: LocalePack = { + locale: 'fr-weekday-offset', + extends: 'en', + date: { + weekdayOffsetPhrases: { 'en huit': 7, 'en quinze': 14 }, + weekdayNames: ['dimanche', 'lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi'], + weekdays: { lundi: 1, mardi: 2 }, + }, + } + expectOkDate('lundi en huit', new Date(2026, 6, 13), 'day', { + locale: 'fr-weekday-offset', + localePacks: [frWeekdayOffset], + }) + expectOkDate('mardi en quinze', new Date(2026, 6, 21), 'day', { + locale: 'fr-weekday-offset', + localePacks: [frWeekdayOffset], + }) + }) + + it('parses after-next and before-last period modifiers from pack data', () => { + expectOkDate('the week after next', new Date(2026, 6, 13), 'week') + expectOkDate('the week before last', new Date(2026, 5, 15), 'week') + const noNow = parseDate('the week after next') + expect(noNow.ok).toBe(false) + expect(!noNow.ok && noNow.issues.some((issue) => issue.code === 'NOW_REQUIRED')).toBe(true) + + const jaDoubleStep: LocalePack = { + locale: 'ja-double-step', + extends: 'en', + date: { + calendarPeriodPhrases: { + 再来週: { modifier: 'afterNext', period: 'week' }, + 先々週: { modifier: 'beforeLast', period: 'week' }, + 再来月: { modifier: 'afterNext', period: 'month' }, + }, + }, + } + expectOkDate('再来週', new Date(2026, 6, 20), 'week', { + now: new Date(2026, 6, 8, 12), + locale: 'ja-double-step', + localePacks: [jaDoubleStep], + }) + expectOkDate('先々週', new Date(2026, 5, 22), 'week', { + now: new Date(2026, 6, 8, 12), + locale: 'ja-double-step', + localePacks: [jaDoubleStep], + }) + expectOkDate('再来月', new Date(2026, 8, 1), 'month', { + now: new Date(2026, 6, 8, 12), + locale: 'ja-double-step', + localePacks: [jaDoubleStep], + }) + }) + + it('parses localized period-edge phrases and month composition', () => { + const esEdges: LocalePack = { + locale: 'es-edges', + extends: 'en', + date: { + fillerWords: ['de'], + months: { julio: 6 }, + periodEdgePhrases: { + 'a finales': { period: 'month', edge: 'end' }, + 'a mediados': { period: 'month', edge: 'mid' }, + 'a principios': { period: 'month', edge: 'start' }, + }, + periodWords: { month: ['mes'] }, + }, + } + expectOkDate('a finales de mes', new Date(2026, 6, 31), 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'es-edges', + localePacks: [esEdges], + }) + expectOkDate('a principios de mes', new Date(2026, 6, 1), 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'es-edges', + localePacks: [esEdges], + }) + expectOkDate('a mediados de julio', new Date(2026, 6, 15), 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'es-edges', + localePacks: [esEdges], + }) + + const frEdges: LocalePack = { + locale: 'fr-edges', + extends: 'en', + date: { + months: { juillet: 6 }, + periodEdgePhrases: { + debut: { period: 'month', edge: 'start' }, + fin: { period: 'month', edge: 'end' }, + mi: { period: 'month', edge: 'mid' }, + }, + }, + } + for (const [input, date] of [ + ['début juillet', new Date(2027, 6, 1)], + ['mi-juillet', new Date(2026, 6, 15)], + ['fin juillet', new Date(2026, 6, 31)], + ] as const) { + expectOkDate(input, date, 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'fr-edges', + localePacks: [frEdges], + }) + } + + const ptEdges: LocalePack = { + locale: 'pt-edges', + extends: 'en', + date: { + fillerWords: ['de', 'do'], + months: { julho: 6 }, + periodEdgePhrases: { + fim: { period: 'month', edge: 'end' }, + inicio: { period: 'month', edge: 'start' }, + meio: { period: 'month', edge: 'mid' }, + 'no comeco': { period: 'month', edge: 'start' }, + }, + periodWords: { month: ['mes'] }, + }, + } + for (const [input, date] of [ + ['início de julho', new Date(2027, 6, 1)], + ['fim de julho', new Date(2026, 6, 31)], + ['meio de julho', new Date(2026, 6, 15)], + ['no começo do mês', new Date(2026, 6, 1)], + ] as const) { + expectOkDate(input, date, 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'pt-edges', + localePacks: [ptEdges], + }) + } + + const zhEdges: LocalePack = { + locale: 'zh-edges', + extends: 'en', + date: { + periodEdgePhrases: { + 月初: { period: 'month', edge: 'start' }, + 月底: { period: 'month', edge: 'end' }, + 年底: { period: 'year', edge: 'end' }, + }, + }, + } + expectOkDate('月底', new Date(2026, 6, 31), 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'zh-edges', + localePacks: [zhEdges], + }) + expectOkDate('月初', new Date(2026, 6, 1), 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'zh-edges', + localePacks: [zhEdges], + }) + expectOkDate('年底', new Date(2026, 11, 31), 'day', { + now: new Date(2026, 6, 8, 12), + locale: 'zh-edges', + localePacks: [zhEdges], + }) + }) }) describe('parseDate absolute dates and times', () => { @@ -340,6 +544,69 @@ describe('parseDate absolute dates and times', () => { } }) + it('parses localized spoken-clock forms from DateVocabPack clock tables', () => { + expectOkDate('quarter of five', new Date(2026, 6, 4, 4, 45), 'minute') + expectOkDate('half seven', new Date(2026, 6, 4, 7, 30), 'minute') + expectOkDate("five o'clock", new Date(2026, 6, 4, 5), 'hour') + + const esClock: LocalePack = { + locale: 'es-clock', + extends: 'en', + numberWords: { ones: { dos: 2, tres: 3, quince: 15 } }, + date: { + clockMinuteWords: { cuarto: 15, media: 30 }, + clockPastWords: ['y'], + clockToWords: ['menos'], + fillerWords: ['las'], + }, + } + expectOkDate('las tres menos cuarto', new Date(2026, 6, 4, 2, 45), 'minute', { + locale: 'es-clock', + localePacks: [esClock], + }) + + const frClock: LocalePack = { + locale: 'fr-clock', + extends: 'en', + numberWords: { ones: { deux: 2, trois: 3 } }, + date: { + clockMinuteWords: { demi: 30, demie: 30, quart: 15 }, + clockPastWords: ['et'], + clockToWords: ['moins'], + fillerWords: ['le'], + unitWords: { heure: 'hour', heures: 'hour' }, + }, + } + expectOkDate('deux heures et quart', new Date(2026, 6, 4, 2, 15), 'minute', { + locale: 'fr-clock', + localePacks: [frClock], + }) + expectOkDate('trois heures moins le quart', new Date(2026, 6, 4, 2, 45), 'minute', { + locale: 'fr-clock', + localePacks: [frClock], + }) + + const ptClock: LocalePack = { + locale: 'pt-clock', + extends: 'en', + numberWords: { ones: { duas: 2, tres: 3, quinze: 15 } }, + date: { + clockMinuteWords: { meia: 30 }, + clockPastWords: ['e'], + clockToWords: ['para as', 'para'], + fillerWords: ['as'], + }, + } + expectOkDate('duas e meia', new Date(2026, 6, 4, 2, 30), 'minute', { + locale: 'pt-clock', + localePacks: [ptClock], + }) + expectOkDate('quinze para as três', new Date(2026, 6, 4, 2, 45), 'minute', { + locale: 'pt-clock', + localePacks: [ptClock], + }) + }) + it('does not misread bare "N to M" as a relative-minute time (that is a range)', () => { // "5 to 6" is 5:55 grammatically but almost always a time SLOT — leave it to // parseDateRange rather than misreading it. Bare 4-digit and decimals too. @@ -421,6 +688,44 @@ describe('parseDuration', () => { expect(month.issues.some((i) => i.code === 'CIVIL_AVERAGE')).toBe(true) }) + it('parses localized duration unit words from DateVocabPack tables', () => { + const esDuration: LocalePack = { + locale: 'es-duration', + extends: 'en', + numberWords: { + andWords: ['y'], + fractionWords: { cuarto: 1 / 4, media: 1 / 2, medio: 1 / 2 }, + ofWords: ['de'], + ones: { dos: 2, un: 1, una: 1 }, + }, + date: { + fillerWords: ['de'], + unitWords: { hora: 'hour', horas: 'hour' }, + }, + } + const esOpts = { locale: 'es-duration', localePacks: [esDuration] } + expectDuration('2 horas', 7200, esOpts) + expectDuration('dos horas', 7200, esOpts) + expectDuration('media hora', 1800, esOpts) + expectDuration('un cuarto de hora', 900, esOpts) + + const ptDuration: LocalePack = { + locale: 'pt-duration', + extends: 'en', + numberWords: { + andWords: ['e'], + fractionWords: { meia: 1 / 2, meio: 1 / 2 }, + ones: { duas: 2, uma: 1, um: 1 }, + }, + date: { + unitWords: { hora: 'hour', horas: 'hour' }, + }, + } + const ptOpts = { locale: 'pt-duration', localePacks: [ptDuration] } + expectDuration('duas horas', 7200, ptOpts) + expectDuration('meia hora', 1800, ptOpts) + }) + it('humanized duration output parses back', () => { for (const style of ['narrow', 'short', 'long', 'natural'] as const) { const text = humanizeDuration(5400, { style }) diff --git a/packages/lingo/src/date/duration.ts b/packages/lingo/src/date/duration.ts index abeadd1..0ee555e 100644 --- a/packages/lingo/src/date/duration.ts +++ b/packages/lingo/src/date/duration.ts @@ -9,6 +9,8 @@ import type { Severity, Span, } from '../core/types' +import { resolveLanguageProfile } from '../locale/profile' +import type { DateOffsetUnit, LocalePack } from '../locale/types' import { parseQuantityExpr } from '../parse/grammar' import { normalizeInput, toSourceSpan } from '../parse/normalize' import { attachSerialization } from '../parse/serialize' @@ -27,6 +29,8 @@ import { DURATION_UNIT_SECONDS } from './vocab' */ export interface DurationOptions { escalate?: Partial> + locale?: string + localePacks?: readonly LocalePack[] messages?: Messages numberFormat?: NumberFormatPolicy registry?: Registry @@ -61,6 +65,14 @@ function registryFor(opts?: DurationOptions): Registry { if (opts?.registry) { return opts.registry } + if (opts?.locale || opts?.localePacks) { + const reg = createRegistry([duration]) + const words = resolveLanguageProfile(opts.localePacks, opts.locale).date?.unitWords ?? {} + for (const word of Object.keys(words)) { + reg.registerUnitAliases('duration', durationUnitId(words[word]!), [word]) + } + return reg + } defaultDurationRegistry ??= createRegistry([duration]) return defaultDurationRegistry } @@ -137,6 +149,8 @@ function parseDurationImpl(text: string, opts?: DurationOptions): DurationResult export function parseUnitDuration(text: string, opts?: DurationOptions): DurationResult | DateFail { const q = parseQuantityExpr(text, { kind: 'duration', + locale: opts?.locale, + localePacks: opts?.localePacks, registry: registryFor(opts), numberFormat: opts?.numberFormat ?? 'auto', messages: opts?.messages, @@ -171,6 +185,18 @@ export function parseUnitDuration(text: string, opts?: DurationOptions): Duratio ) } +function durationUnitId(unit: DateOffsetUnit): string { + return unit === 'minute' + ? 'min' + : unit === 'month' + ? 'mo' + : unit === 'year' + ? 'yr' + : unit === 'week' + ? 'wk' + : unit[0]! +} + function parseIsoDuration( source: string, text: string, diff --git a/packages/lingo/src/date/parse.ts b/packages/lingo/src/date/parse.ts index 3999220..068cd94 100644 --- a/packages/lingo/src/date/parse.ts +++ b/packages/lingo/src/date/parse.ts @@ -529,6 +529,8 @@ function parseAnchoredDurationRange( const anchorEnd = normStart + source.length const duration = parseUnitDuration(p.text.slice(durationStart, durationEnd), { escalate: p.opts.escalate, + locale: p.opts.locale, + localePacks: p.opts.localePacks, messages: p.opts.messages, strictness: p.opts.strictness, }) diff --git a/packages/lingo/src/date/relative.ts b/packages/lingo/src/date/relative.ts index 992ad7a..fd49f74 100644 --- a/packages/lingo/src/date/relative.ts +++ b/packages/lingo/src/date/relative.ts @@ -38,6 +38,31 @@ interface OffsetPart { value: number } +const EN_DAY_PART_WORDS = { + morning: { hour: 9 }, + afternoon: { hour: 15 }, + evening: { hour: 19 }, + night: { hour: 21 }, +} + +const EN_WEEKDAY_OFFSET_PHRASES = { + 'a week on': 7, + 'a week': 7, + fortnight: 14, + 'week on': 7, + week: 7, +} + +const RELATIVE_MODIFIERS = ['this', 'next', 'last', 'afterNext', 'beforeLast'] as const +const PERIODS = ['week', 'month', 'year'] as const +const PERIOD_DELTAS: Record = { + this: 0, + next: 1, + last: -1, + afterNext: 2, + beforeLast: -2, +} + export function parseDateOnly( p: P, start: number, @@ -92,6 +117,10 @@ function parseDeictic(p: P, start: number, end: number): CoreDate | null { if (dayTime) { return dayTimeCore(p, today, dayTime, start, end) } + const dayPart = parseDayPartCompound(p, source, today, start, end) + if (dayPart) { + return dayPart + } const dayOffset = dateDayOffsets(p)[source] if (dayOffset !== undefined) { return core(addCalendar(today, { days: dayOffset }), 'day', knownFor('day'), start, end) @@ -268,9 +297,13 @@ function offsetCore( } function parseWeekday(p: P, start: number, end: number): CoreDate | null { - const source = stripDateFillers(p, p.text.slice(start, end).toLowerCase()) + let source = stripDateFillers(p, p.text.slice(start, end).toLowerCase()) + const offset = extractWeekdayOffset(p, source) + if (offset) { + source = offset.source + } const words = source.split(/\s+/) - let modifier: 'bare' | 'this' | 'next' | 'last' = 'bare' + let modifier: 'bare' | RelativeModifier = 'bare' let dayWord = words[0] const withModifier = matchWordModifier(p, source) if (withModifier) { @@ -310,6 +343,9 @@ function parseWeekday(p: P, start: number, end: number): CoreDate | null { } else if (modifier === 'last') { const diff = backwardDiff(today.getDay(), weekday) || 7 date = addCalendar(today, { days: -diff }) + } else if (modifier === 'afterNext' || modifier === 'beforeLast') { + const days = modifier === 'afterNext' ? 14 : -14 + date = dateInWeek(addCalendar(startOfWeek(today, p.weekStart), { days }), weekday, p.weekStart) } else { const diff = p.forwardDates ? forwardDiff(today.getDay(), weekday) @@ -319,15 +355,50 @@ function parseWeekday(p: P, start: number, end: number): CoreDate | null { issue(p, 'WEEKDAY_ASSUMED_NEXT', { weekday: dateWeekdayNames(p)[weekday]! }, start, end), ) } - return core(date, 'day', [...knownFor('day'), 'weekday'], start, end, issues, alternatives) + const result = core( + date, + 'day', + [...knownFor('day'), 'weekday'], + start, + end, + issues, + alternatives, + ) + return offset ? applyWeekdayOffset(result, offset.days) : result +} + +function extractWeekdayOffset(p: P, source: string): { days: number; source: string } | null { + const phrases = dateWeekdayOffsetPhrases(p) + for (const phrase of Object.keys(phrases)) { + if (source.endsWith(` ${phrase}`)) { + return { source: source.slice(0, -phrase.length - 1).trim(), days: phrases[phrase]! } + } + if (source.startsWith(`${phrase} `)) { + return { source: source.slice(phrase.length + 1).trim(), days: phrases[phrase]! } + } + } + return null +} + +function applyWeekdayOffset(result: CoreDate, days: number): CoreDate { + result.date = addCalendar(result.date, { days }) + result.issues = result.issues.filter((it) => it.code !== 'WEEKDAY_ASSUMED_NEXT') + if (result.alternatives) { + result.alternatives = result.alternatives.map((alt) => ({ + ...alt, + date: addCalendar(alt.date, { days }), + })) + } + return result } function matchWordModifier( p: P, source: string, -): { modifier: 'this' | 'next' | 'last'; word: string } | null { - for (const modifier of ['this', 'next', 'last'] as const) { - for (const modWord of dateModifiers(p)[modifier]) { +): { modifier: RelativeModifier; word: string } | null { + const modifiers = dateModifiers(p) + for (const modifier of RELATIVE_MODIFIERS) { + for (const modWord of modifiers[modifier]) { if (source.startsWith(`${modWord} `)) { return { modifier, word: source.slice(modWord.length + 1) } } @@ -355,6 +426,10 @@ function parseCalendarPeriod(p: P, start: number, end: number): CoreDate | null if (localePeriod) { return periodCore(p, localePeriod.modifier, localePeriod.period, start, end) } + const periodEdge = matchPeriodEdge(p, source, start, end) + if (periodEdge) { + return periodEdge + } const monthRel = matchMonthModifier(p, source) if (monthRel) { const { modifier: mod, monthWord } = monthRel @@ -386,9 +461,6 @@ function parseCalendarPeriod(p: P, start: number, end: number): CoreDate | null return middlePeriodCore(p, mod, target, start, end) } const base = periodCore(p, mod, target, start, end) - if (!base) { - return null - } if (kind === 'end') { return endPeriodCore(p, base.date, target, start, end) } @@ -411,12 +483,12 @@ function parseCalendarPeriod(p: P, start: number, end: number): CoreDate | null function periodCore( p: P, - mod: 'this' | 'next' | 'last', + mod: RelativeModifier, period: 'week' | 'month' | 'year', start: number, end: number, ): CoreDate { - const delta = mod === 'next' ? 1 : mod === 'last' ? -1 : 0 + const delta = PERIOD_DELTAS[mod] let date: Date if (period === 'week') { date = addCalendar(startOfWeek(p.now, p.weekStart), { days: delta * 7 }) @@ -444,7 +516,7 @@ function monthPeriodCore( function middlePeriodCore( p: P, - mod: 'this' | 'next' | 'last', + mod: RelativeModifier, period: 'week' | 'month' | 'year', start: number, end: number, @@ -465,6 +537,23 @@ function middlePeriodCore( return core(new Date(base.getFullYear(), 6, 2), 'day', knownFor('day'), start, end) } +function periodEdgeCore( + p: P, + mod: RelativeModifier, + period: 'week' | 'month' | 'year', + edge: 'start' | 'mid' | 'end', + start: number, + end: number, +): CoreDate { + if (edge === 'mid') { + return middlePeriodCore(p, mod, period, start, end) + } + const base = periodCore(p, mod, period, start, end) + return edge === 'end' + ? endPeriodCore(p, base.date, period, start, end) + : core(base.date, 'day', knownFor('day'), start, end) +} + function endPeriodCore( p: P, base: Date, @@ -487,6 +576,65 @@ function endPeriodCore( return core(new Date(base.getFullYear(), 11, 31), 'day', knownFor('day'), start, end) } +function matchPeriodEdge(p: P, source: string, start: number, end: number): CoreDate | null { + const phrases = p.profile.date?.periodEdgePhrases + if (!phrases) { + return null + } + for (const variant of source === source.replace(/-/g, ' ') + ? [source] + : [source, source.replace(/-/g, ' ')]) { + const exact = phrases[variant] + if (exact) { + return periodEdgeCore(p, 'this', exact.period, exact.edge, start, end) + } + for (const phrase of Object.keys(phrases)) { + if (!variant.startsWith(`${phrase} `)) { + continue + } + const target = variant.slice(phrase.length + 1).trim() + const edge = phrases[phrase]!.edge + const period = periodWord(p, target) + if (period) { + return periodEdgeCore(p, 'this', period, edge, start, end) + } + const month = dateMonths(p)[target] + if (month !== undefined) { + return monthEdgeCore(p, month, edge, start, end) + } + } + } + return null +} + +function periodWord(p: P, word: string): PeriodUnit | null { + const periods = datePeriodWords(p) + for (const period of PERIODS) { + if (periods[period].includes(word)) { + return period + } + } + return null +} + +function monthEdgeCore( + p: P, + month: number, + edge: 'start' | 'mid' | 'end', + start: number, + end: number, +): CoreDate | null { + if (edge === 'end') { + let date = new Date(p.now.getFullYear(), month + 1, 0) + if (p.forwardDates && date.getTime() < startOfDay(p.now).getTime()) { + date = new Date(p.now.getFullYear() + 1, month + 1, 0) + } + return core(date, 'day', knownFor('day'), start, end) + } + const day = edge === 'mid' ? 15 : 1 + return monthDayCore(p, month, day, undefined, 'day', start, end) +} + function wordAt(p: P, i: number): string | null { const t = p.tokens[i] return t?.type === 'word' ? t.text.toLowerCase() : null @@ -570,6 +718,61 @@ function dateDayTimePhrases(p: P): Record { return p.profile.date?.dayTimePhrases ?? EN_DAY_TIME_PHRASES } +function parseDayPartCompound( + p: P, + source: string, + today: Date, + start: number, + end: number, +): CoreDate | null { + const parts = dateDayPartWords(p) + for (const phrase of Object.keys(parts)) { + if (!source.endsWith(` ${phrase}`)) { + continue + } + const partStart = end - phrase.length + const { end: anchorEnd } = trimRange(p.text, start, partStart) + if (anchorEnd <= start) { + continue + } + const anchor = p.text.slice(start, anchorEnd).toLowerCase() + const dayOffset = dateDayOffsets(p)[anchor] + const part = parts[phrase]! + const grain = part.grain ?? 'hour' + if (dayOffset !== undefined) { + return core( + withTime(addCalendar(today, { days: dayOffset }), part.hour), + grain, + [...knownFor(grain), 'implied-hour'], + start, + end, + ) + } + const weekday = parseWeekday(p, start, anchorEnd) + if (weekday) { + const date = withTime(weekday.date, part.hour) + return core( + date, + grain, + [...knownFor(grain), 'weekday', 'implied-hour'], + start, + end, + weekday.issues, + weekday.alternatives?.map((alt) => ({ ...alt, date: withTime(alt.date, part.hour) })), + ) + } + } + return null +} + +function dateDayPartWords(p: P): Record { + return p.profile.date?.dayPartWords ?? EN_DAY_PART_WORDS +} + +function dateWeekdayOffsetPhrases(p: P): Record { + return p.profile.date?.weekdayOffsetPhrases ?? EN_WEEKDAY_OFFSET_PHRASES +} + function dayTimeCore( p: P, today: Date, @@ -608,6 +811,8 @@ function dateModifiers(p: P): Record { this: wordsOr(p.profile.date?.modifiers?.this, EN_MODIFIERS.this), next: wordsOr(p.profile.date?.modifiers?.next, EN_MODIFIERS.next), last: wordsOr(p.profile.date?.modifiers?.last, EN_MODIFIERS.last), + afterNext: wordsOr(p.profile.date?.modifiers?.afterNext, EN_MODIFIERS.afterNext), + beforeLast: wordsOr(p.profile.date?.modifiers?.beforeLast, EN_MODIFIERS.beforeLast), } } @@ -641,8 +846,8 @@ function isPrefixAt(p: P, first: number, phrases: readonly string[] | undefined) } function stripDateFillers(p: P, source: string): string { - const fillers = p.profile.date?.fillerWords - if (!(fillers && fillers.length > 0)) { + const fillers = p.profile.date?.fillerWords ?? ['the'] + if (fillers.length === 0) { return source } const fillerSet = new Set(fillers) @@ -657,9 +862,10 @@ function matchPeriodModifier( source: string, ): { modifier: RelativeModifier; period: PeriodUnit } | null { const periods = datePeriodWords(p) - for (const modifier of ['this', 'next', 'last'] as const) { - for (const modWord of dateModifiers(p)[modifier]) { - for (const period of ['week', 'month', 'year'] as const) { + const modifiers = dateModifiers(p) + for (const modifier of RELATIVE_MODIFIERS) { + for (const modWord of modifiers[modifier]) { + for (const period of PERIODS) { for (const periodWord of periods[period]) { if (source === `${modWord} ${periodWord}` || source === `${periodWord} ${modWord}`) { return { modifier, period } @@ -675,9 +881,11 @@ function matchMonthModifier( p: P, source: string, ): { modifier: 'next' | 'last'; monthWord: string } | null { + const modifiers = dateModifiers(p) + const months = Object.keys(dateMonths(p)) for (const modifier of ['next', 'last'] as const) { - for (const modWord of dateModifiers(p)[modifier]) { - for (const monthWord of Object.keys(dateMonths(p))) { + for (const modWord of modifiers[modifier]) { + for (const monthWord of months) { if (source === `${modWord} ${monthWord}` || source === `${monthWord} ${modWord}`) { return { modifier, monthWord } } diff --git a/packages/lingo/src/date/time.ts b/packages/lingo/src/date/time.ts index 623865c..41ee11b 100644 --- a/packages/lingo/src/date/time.ts +++ b/packages/lingo/src/date/time.ts @@ -1,7 +1,7 @@ import type { LingoIssue } from '../core/types' import type { DateGrain } from './parse' import { issue, type P, trimRange } from './state' -import { TIME_ALIASES, type TimeAlias } from './vocab' +import { TIME_ALIASES, type TimeAlias, UNIT_WORDS } from './vocab' import { type DateZone, stripTrailingZone } from './zone' export interface TimeCore { @@ -136,7 +136,7 @@ export function parseTimeCore(p: P, source: string, issues: LingoIssue[]): TimeC } return { hour, minute, second: 0, grain: 'minute', issues } } - const relative = parseRelativeMinutes(lower, issues) + const relative = parseRelativeMinutes(p, lower, issues) if (relative) { return relative } @@ -174,86 +174,200 @@ function aliasTime(alias: TimeAlias, issues: LingoIssue[]): TimeCore { } } -const TIME_NUM_WORDS: Record = { - one: 1, - two: 2, - three: 3, - four: 4, - five: 5, - six: 6, - seven: 7, - eight: 8, - nine: 9, - ten: 10, - eleven: 11, - twelve: 12, - thirteen: 13, - fourteen: 14, - fifteen: 15, - sixteen: 16, - seventeen: 17, - eighteen: 18, - nineteen: 19, - twenty: 20, - 'twenty-five': 25, - 'twenty five': 25, +const EN_CLOCK_MINUTES = { half: 30, quarter: 15 } +const EN_CLOCK_PAST = ['past', 'after'] +const EN_CLOCK_TO = ['to', 'till', 'til', 'before', 'of'] + +interface ClockRead { + digit?: boolean + next: number + value: number } -function wordOrDigit(token: string): number | null { - if (/^\d{1,2}$/.test(token)) { - return Number(token) +function parseRelativeMinutes(p: P, lower: string, issues: LingoIssue[]): TimeCore | null { + const words = lower.split(/\s+/).filter(Boolean) + const pos = skipClockFillers(p, words, 0) + const first = words[pos] + if (first && clockMinuteWords(p)[first] === 30) { + const hour = readClockHour(p, words, pos + 1) + if (hour && atEnd(p, words, hour.next)) { + return clock(hour.value, 30, issues) + } + } + + const minute = readClockMinute(p, words, pos) + if (minute) { + const past = readPhrase(words, minute.next, clockPastWords(p)) + if (past >= 0) { + const hour = readClockHour(p, words, past) + if (hour && atEnd(p, words, hour.next)) { + return clock(hour.value, minute.value, issues) + } + } + const to = readPhrase(words, minute.next, clockToWords(p)) + if (to >= 0 && !minute.digit) { + const hour = readClockHour(p, words, to) + if (hour && atEnd(p, words, hour.next)) { + return clock((hour.value + 23) % 24, 60 - minute.value, issues) + } + } } - return TIME_NUM_WORDS[token] ?? null -} -function minuteAmount(token: string): number | null { - if (token === 'quarter') { - return 15 + const hour = readClockHour(p, words, pos) + if (!hour) { + return null + } + const next = skipHourUnit(p, words, hour.next) + if (words[next] === "o'clock" || words[next] === 'oclock') { + return atEnd(p, words, next + 1) ? clock(hour.value, 0, issues, 'hour') : null } - if (token === 'half') { - return 30 + const past = readPhrase(words, next, clockPastWords(p)) + if (past >= 0) { + const amount = readClockMinute(p, words, past) + return amount && atEnd(p, words, amount.next) ? clock(hour.value, amount.value, issues) : null } - const n = wordOrDigit(token) - return n !== null && n >= 1 && n <= 59 ? n : null + const to = readPhrase(words, next, clockToWords(p)) + if (to >= 0) { + const amount = readClockMinute(p, words, to) + return amount && !amount.digit && atEnd(p, words, amount.next) + ? clock((hour.value + 23) % 24, 60 - amount.value, issues) + : null + } + if (next > hour.next) { + const amount = readClockMinute(p, words, next) + if (amount && atEnd(p, words, amount.next)) { + return clock(hour.value, amount.value, issues) + } + } + return null } -function hourAmount(token: string): number | null { - const n = wordOrDigit(token) - return n !== null && n >= 1 && n <= 23 ? n : null +function clock( + hour: number, + minute: number, + issues: LingoIssue[], + grain: TimeCore['grain'] = 'minute', +): TimeCore { + return { hour: hour % 24, minute, second: 0, grain, issues } } -/** - * Spoken relative-minute times: "quarter past 5" (5:15), "half past 3" (3:30), - * "quarter to 6" (5:45), "twenty past 4", "ten to 6", and British "half 5" (5:30). - */ -function parseRelativeMinutes(lower: string, issues: LingoIssue[]): TimeCore | null { - const british = /^half\s+(\d{1,2}|[a-z-]+)$/.exec(lower) - if (british) { - const h = hourAmount(british[1]!) - if (h !== null && h >= 1 && h <= 12) { - return { hour: h, minute: 30, second: 0, grain: 'minute', issues } +function readClockMinute(p: P, words: readonly string[], pos: number): ClockRead | null { + pos = skipClockFillers(p, words, pos) + const named = readRecord(words, pos, clockMinuteWords(p)) + if (named && named.value >= 1 && named.value <= 59) { + return named + } + const n = readClockNumber(p, words, pos, 59) + return n && n.value >= 1 ? n : null +} + +function readClockHour(p: P, words: readonly string[], pos: number): ClockRead | null { + pos = skipClockFillers(p, words, pos) + const alias = readAlias(p, words, pos) + return alias ?? readClockNumber(p, words, pos, 23) +} + +function readClockNumber( + p: P, + words: readonly string[], + pos: number, + max: number, +): ClockRead | null { + const phrase = words[pos] + const value = phrase ? clockNumber(p, phrase) : null + return value !== null && value >= 0 && value <= max + ? { value, next: pos + 1, digit: /^\d/.test(phrase!) } + : null +} + +function clockNumber(p: P, phrase: string): number | null { + if (/^\d{1,2}$/.test(phrase)) { + return Number(phrase) + } + const n = p.profile.numberWords + const direct = n.composed?.[phrase] ?? n.ones[phrase] ?? n.tens[phrase] + if (direct !== undefined) { + return direct + } + const parts = phrase.split('-') + if (parts.length === 2) { + const ten = n.tens[parts[0]!] + const one = n.ones[parts[1]!] + if (ten !== undefined && one !== undefined && one < 10) { + return ten + one } } - const m = - /^(quarter|half|\d{1,2}|[a-z-]+)\s+(past|after|to|till|til|before)\s+(\d{1,2}|[a-z-]+)$/.exec( - lower, - ) - if (!m) { - return null + return null +} + +function readAlias(p: P, words: readonly string[], pos: number): ClockRead | null { + const aliases = timeAliases(p) + for (const key of Object.keys(aliases)) { + const len = matchPhrase(words, pos, key) + if (len > 0) { + return { value: aliases[key]!.hour, next: pos + len } + } } - const amount = minuteAmount(m[1]!) - const hour = hourAmount(m[3]!) - if (amount === null || hour === null) { - return null + return null +} + +function readRecord( + words: readonly string[], + pos: number, + record: Record, +): ClockRead | null { + for (const key of Object.keys(record)) { + const len = matchPhrase(words, pos, key) + if (len > 0) { + return { value: record[key]!, next: pos + len } + } } - if (m[2] === 'past' || m[2] === 'after') { - return { hour: hour % 24, minute: amount, second: 0, grain: 'minute', issues } + return null +} + +function readPhrase(words: readonly string[], pos: number, phrases: readonly string[]): number { + for (const phrase of phrases) { + const len = matchPhrase(words, pos, phrase) + if (len > 0) { + return pos + len + } } - // to/till/before: `amount` minutes before `hour`. Require a minute WORD - // (quarter/half/ten/…), never a bare digit, so "5 to 6"/"9 to 5" stay a time - // RANGE (parseDateRange) instead of misreading as 5:55 / 4:51. - if (/^\d/.test(m[1]!)) { - return null + return -1 +} + +function matchPhrase(words: readonly string[], pos: number, phrase: string): number { + const parts = phrase.split(/\s+/) + return parts.every((word, index) => words[pos + index] === word) ? parts.length : 0 +} + +function skipClockFillers(p: P, words: readonly string[], pos: number): number { + while (words[pos] && p.profile.date?.fillerWords?.includes(words[pos]!)) { + pos++ } - return { hour: (hour + 23) % 24, minute: 60 - amount, second: 0, grain: 'minute', issues } + return pos +} + +function skipHourUnit(p: P, words: readonly string[], pos: number): number { + pos = skipClockFillers(p, words, pos) + return dateUnitWords(p)[words[pos]!] === 'hour' ? skipClockFillers(p, words, pos + 1) : pos +} + +function atEnd(p: P, words: readonly string[], pos: number): boolean { + return skipClockFillers(p, words, pos) === words.length +} + +function clockMinuteWords(p: P): Record { + return p.profile.date?.clockMinuteWords ?? EN_CLOCK_MINUTES +} + +function clockPastWords(p: P): readonly string[] { + return p.profile.date?.clockPastWords ?? EN_CLOCK_PAST +} + +function clockToWords(p: P): readonly string[] { + return p.profile.date?.clockToWords ?? EN_CLOCK_TO +} + +function dateUnitWords(p: P): Record { + return p.profile.date?.unitWords ?? UNIT_WORDS } diff --git a/packages/lingo/src/date/vocab.ts b/packages/lingo/src/date/vocab.ts index f76e4e0..b39fd61 100644 --- a/packages/lingo/src/date/vocab.ts +++ b/packages/lingo/src/date/vocab.ts @@ -2,7 +2,7 @@ // stay additive — new languages add tables, they never edit grammar code. export type OffsetUnit = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year' -export type RelativeModifier = 'this' | 'next' | 'last' +export type RelativeModifier = 'this' | 'next' | 'last' | 'afterNext' | 'beforeLast' export type PeriodUnit = 'week' | 'month' | 'year' export type TimeAlias = { grain?: 'hour' | 'minute' | 'second' @@ -156,6 +156,8 @@ export const MODIFIERS: Record = { this: ['this'], next: ['next'], last: ['last'], + afterNext: ['after next'], + beforeLast: ['before last'], } export const PERIOD_WORDS: Record = { diff --git a/packages/lingo/src/locale/detect.ts b/packages/lingo/src/locale/detect.ts index 83036b3..7452195 100644 --- a/packages/lingo/src/locale/detect.ts +++ b/packages/lingo/src/locale/detect.ts @@ -64,16 +64,20 @@ export function scoreProfile( const unique = !uniqueAgainst?.has(word) const standalone = isStandaloneWord(tokens, i) if ( - (unique && profile.numberWords.ones[word] !== undefined) || - (unique && profile.numberWords.tens[word] !== undefined) || - (unique && profile.numberWords.scales[word] !== undefined) + unique && + (profile.numberWords.ones[word] !== undefined || + profile.numberWords.tens[word] !== undefined || + profile.numberWords.scales[word] !== undefined || + profile.numberWords.bareScales?.[word] !== undefined || + profile.numberWords.composed?.[word] !== undefined) ) { score += 4 } if ( unique && (profile.numberWords.fuzzyAmounts[word] || - profile.numberWords.fractionWords[word] !== undefined) + profile.numberWords.fractionWords[word] !== undefined || + profile.numberWords.decimalWords?.has(word)) ) { score += 3 } @@ -170,8 +174,13 @@ function profileWords(profile: LanguageProfile): ReadonlySet { addRecordKeys(words, profile.numberWords.ones) addRecordKeys(words, profile.numberWords.tens) addRecordKeys(words, profile.numberWords.scales) + addRecordKeys(words, profile.numberWords.bareScales) + addRecordKeys(words, profile.numberWords.composed) addRecordKeys(words, profile.numberWords.fuzzyAmounts) addRecordKeys(words, profile.numberWords.fractionWords) + if (profile.numberWords.decimalWords) { + addSet(words, profile.numberWords.decimalWords) + } addSet(words, profile.numberWords.andWords) addSet(words, profile.numberWords.articles) addSet(words, profile.numberWords.dozenWords) diff --git a/packages/lingo/src/locale/en-core.ts b/packages/lingo/src/locale/en-core.ts index 0871451..d6b7ff7 100644 --- a/packages/lingo/src/locale/en-core.ts +++ b/packages/lingo/src/locale/en-core.ts @@ -8,6 +8,7 @@ export const enCore: LocalePack = { system: 'us', }, grammar: { + approximatePhrases: [], approximateWords: new Set([ 'about', 'around', @@ -113,6 +114,7 @@ export const enCore: LocalePack = { numberWords: { andWords: new Set(['and']), articles: new Set(['a', 'an']), + decimalWords: new Set(['point']), dozenWords: new Set(['dozen']), fractionWords: { half: 1 / 2, diff --git a/packages/lingo/src/locale/profile.ts b/packages/lingo/src/locale/profile.ts index 4c9f6f5..234ceaa 100644 --- a/packages/lingo/src/locale/profile.ts +++ b/packages/lingo/src/locale/profile.ts @@ -13,6 +13,52 @@ import type { export const DEFAULT_LOCALE_PACKS: readonly LocalePack[] = [enCore] +const GRAMMAR_SET_KEYS = [ + 'approximateWords', + 'compoundJoinWords', + 'compoundMinusWords', + 'compoundPlusWords', + 'conversionWords', + 'exactWords', + 'globalFillers', + 'qualifierArticleFollowers', + 'qualifierArticleWords', + 'qualifierFillers', + 'qualifierSkipAfterApprox', + 'qualifierSoftenerWords', + 'rangeAlternativeWords', + 'rangeAndWords', + 'rangeBetweenWords', + 'rangeFromWords', + 'rangeSeparatorWords', + 'trailingApproxWords', + 'trailingOkWords', +] as const + +const GRAMMAR_LIST_KEYS = [ + 'approximatePhrases', + 'phraseWords', + 'qualifierSoftenerPhrases', + 'trailingApproxPhrases', +] as const + +const LONGEST_FIRST_PHRASE_KEYS: readonly (keyof GrammarWords)[] = [ + 'approximatePhrases', + 'qualifierSoftenerPhrases', + 'trailingApproxPhrases', +] + +const NUMBER_SET_KEYS = ['andWords', 'articles', 'dozenWords', 'negativeWords', 'ofWords'] as const +const NUMBER_RECORD_KEYS = ['fractionWords', 'fuzzyAmounts', 'ones', 'scales', 'tens'] as const +const DATE_MODIFIER_KEYS = ['this', 'next', 'last', 'afterNext', 'beforeLast'] as const +const DATE_PERIOD_KEYS = ['week', 'month', 'year'] as const +const DATE_RELATIVE_KEYS = [ + 'anchorWords', + 'futurePrefixes', + 'pastPrefixes', + 'pastSuffixes', +] as const + export function resolveLanguageProfile( packs: readonly LocalePack[] = DEFAULT_LOCALE_PACKS, locale?: string, @@ -87,86 +133,47 @@ function emptyProfile(locale: string): LanguageProfile { } function emptyGrammar(): GrammarWords { - return { - approximateWords: new Set(), - boundPhrases: [], - compoundJoinWords: new Set(), - compoundMinusWords: new Set(), - compoundPlusWords: new Set(), - conversionWords: new Set(), - exactWords: new Set(), - globalFillers: new Set(), - phraseWords: [], - qualifierArticleFollowers: new Set(), - qualifierArticleWords: new Set(), - qualifierFillers: new Set(), - qualifierSkipAfterApprox: new Set(), - qualifierSoftenerPhrases: [], - qualifierSoftenerWords: new Set(), - rangeAlternativeWords: new Set(), - rangeAndWords: new Set(), - rangeBetweenWords: new Set(), - rangeFromWords: new Set(), - rangeSeparatorWords: new Set(), - trailingApproxPhrases: [], - trailingApproxWords: new Set(), - trailingOkWords: new Set(), + const result = { boundPhrases: [] } as unknown as GrammarWords + for (const key of GRAMMAR_SET_KEYS) { + result[key] = new Set() + } + for (const key of GRAMMAR_LIST_KEYS) { + result[key] = [] } + return result } function emptyNumberWords(): NumberWordTables { - return { - andWords: new Set(), - articles: new Set(), - dozenWords: new Set(), - fractionWords: {}, - fuzzyAmounts: {}, - negativeWords: new Set(), - ofWords: new Set(), - ones: {}, - scales: {}, - tens: {}, + const result = {} as NumberWordTables + for (const key of NUMBER_SET_KEYS) { + result[key] = new Set() + } + for (const key of NUMBER_RECORD_KEYS) { + result[key] = {} } + return result } function mergeGrammar(base: GrammarWords, overlay?: Partial): GrammarWords { if (!overlay) { return base } - return { - approximateWords: mergeSet(base.approximateWords, overlay.approximateWords), + const result = { boundPhrases: mergeBoundPhrases(base.boundPhrases, overlay.boundPhrases), - compoundJoinWords: mergeSet(base.compoundJoinWords, overlay.compoundJoinWords), - compoundMinusWords: mergeSet(base.compoundMinusWords, overlay.compoundMinusWords), - compoundPlusWords: mergeSet(base.compoundPlusWords, overlay.compoundPlusWords), - conversionWords: mergeSet(base.conversionWords, overlay.conversionWords), - exactWords: mergeSet(base.exactWords, overlay.exactWords), - globalFillers: mergeSet(base.globalFillers, overlay.globalFillers), - phraseWords: mergeList(base.phraseWords, overlay.phraseWords), - qualifierArticleFollowers: mergeSet( - base.qualifierArticleFollowers, - overlay.qualifierArticleFollowers, - ), - qualifierArticleWords: mergeSet(base.qualifierArticleWords, overlay.qualifierArticleWords), - qualifierFillers: mergeSet(base.qualifierFillers, overlay.qualifierFillers), - qualifierSkipAfterApprox: mergeSet( - base.qualifierSkipAfterApprox, - overlay.qualifierSkipAfterApprox, - ), - qualifierSoftenerPhrases: mergeList( - base.qualifierSoftenerPhrases, - overlay.qualifierSoftenerPhrases, - ), - qualifierSoftenerWords: mergeSet(base.qualifierSoftenerWords, overlay.qualifierSoftenerWords), - rangeAlternativeWords: mergeSet(base.rangeAlternativeWords, overlay.rangeAlternativeWords), - rangeAndWords: mergeSet(base.rangeAndWords, overlay.rangeAndWords), - rangeBetweenWords: mergeSet(base.rangeBetweenWords, overlay.rangeBetweenWords), - rangeFromWords: mergeSet(base.rangeFromWords, overlay.rangeFromWords), - rangeSeparatorWords: mergeSet(base.rangeSeparatorWords, overlay.rangeSeparatorWords), - trailingApproxPhrases: mergeList(base.trailingApproxPhrases, overlay.trailingApproxPhrases), - trailingApproxWords: mergeSet(base.trailingApproxWords, overlay.trailingApproxWords), - trailingOkWords: mergeSet(base.trailingOkWords, overlay.trailingOkWords), + } as GrammarWords + for (const key of GRAMMAR_SET_KEYS) { + result[key] = mergeSet(base[key], overlay[key]) + } + for (const key of GRAMMAR_LIST_KEYS) { + result[key] = mergeList(base[key], overlay[key]) + } + // Sort phrase lists longest-first so eatAnyPhrase matches greedily. + for (const key of LONGEST_FIRST_PHRASE_KEYS) { + ;(result as unknown as Record)[key] = sortLongestFirst( + result[key] as readonly string[], + ) } + return result } function mergeNumberWords( @@ -176,18 +183,24 @@ function mergeNumberWords( if (!overlay) { return base } - return { - andWords: mergeSet(base.andWords, overlay.andWords), - articles: mergeSet(base.articles, overlay.articles), - dozenWords: mergeSet(base.dozenWords, overlay.dozenWords), - fractionWords: { ...base.fractionWords, ...overlay.fractionWords }, - fuzzyAmounts: { ...base.fuzzyAmounts, ...overlay.fuzzyAmounts }, - negativeWords: mergeSet(base.negativeWords, overlay.negativeWords), - ofWords: mergeSet(base.ofWords, overlay.ofWords), - ones: { ...base.ones, ...overlay.ones }, - scales: { ...base.scales, ...overlay.scales }, - tens: { ...base.tens, ...overlay.tens }, + const result = {} as NumberWordTables + for (const key of NUMBER_SET_KEYS) { + result[key] = mergeSet(base[key], overlay[key]) + } + const records = result as unknown as Record + for (const key of NUMBER_RECORD_KEYS) { + records[key] = { ...base[key], ...overlay[key] } + } + for (const key of ['bareScales', 'composed'] as const) { + if (overlay[key] ?? base[key]) { + result[key] = { ...(base[key] ?? {}), ...(overlay[key] ?? {}) } + } + } + const decimalWords = overlay.decimalWords ?? base.decimalWords + if (decimalWords) { + result.decimalWords = mergeSet(base.decimalWords ?? new Set(), overlay.decimalWords) } + return result } function mergeDate( @@ -198,10 +211,10 @@ function mergeDate( return } return { - calendarPeriodPhrases: { - ...(base?.calendarPeriodPhrases ?? {}), - ...overlay?.calendarPeriodPhrases, - }, + calendarPeriodPhrases: mergeRecord(base?.calendarPeriodPhrases, overlay?.calendarPeriodPhrases), + clockMinuteWords: mergeRecord(base?.clockMinuteWords, overlay?.clockMinuteWords), + clockPastWords: mergeList(base?.clockPastWords ?? [], overlay?.clockPastWords), + clockToWords: mergeList(base?.clockToWords ?? [], overlay?.clockToWords), compactOffset: base?.compactOffset || overlay?.compactOffset ? { @@ -213,64 +226,52 @@ function mergeDate( base?.compactOffset?.pastSuffixes ?? [], overlay?.compactOffset?.pastSuffixes, ), - unitWords: { - ...(base?.compactOffset?.unitWords ?? {}), - ...overlay?.compactOffset?.unitWords, - }, - } - : undefined, - dayOffsets: { ...(base?.dayOffsets ?? {}), ...overlay?.dayOffsets }, - dayTimePhrases: { ...(base?.dayTimePhrases ?? {}), ...overlay?.dayTimePhrases }, - durationUnitSeconds: { ...(base?.durationUnitSeconds ?? {}), ...overlay?.durationUnitSeconds }, - fillerWords: mergeList(base?.fillerWords ?? [], overlay?.fillerWords), - modifiers: - base?.modifiers || overlay?.modifiers - ? { - this: mergeList(base?.modifiers?.this ?? [], overlay?.modifiers?.this), - next: mergeList(base?.modifiers?.next ?? [], overlay?.modifiers?.next), - last: mergeList(base?.modifiers?.last ?? [], overlay?.modifiers?.last), - } - : undefined, - months: { ...(base?.months ?? {}), ...overlay?.months }, - periodWords: - base?.periodWords || overlay?.periodWords - ? { - week: mergeList(base?.periodWords?.week ?? [], overlay?.periodWords?.week), - month: mergeList(base?.periodWords?.month ?? [], overlay?.periodWords?.month), - year: mergeList(base?.periodWords?.year ?? [], overlay?.periodWords?.year), - } - : undefined, - relative: - base?.relative || overlay?.relative - ? { - anchorWords: mergeList( - base?.relative?.anchorWords ?? [], - overlay?.relative?.anchorWords, - ), - futurePrefixes: mergeList( - base?.relative?.futurePrefixes ?? [], - overlay?.relative?.futurePrefixes, - ), - pastPrefixes: mergeList( - base?.relative?.pastPrefixes ?? [], - overlay?.relative?.pastPrefixes, - ), - pastSuffixes: mergeList( - base?.relative?.pastSuffixes ?? [], - overlay?.relative?.pastSuffixes, + unitWords: mergeRecord( + base?.compactOffset?.unitWords, + overlay?.compactOffset?.unitWords, ), } : undefined, - subunit: { ...(base?.subunit ?? {}), ...overlay?.subunit }, - timeAliases: { ...(base?.timeAliases ?? {}), ...overlay?.timeAliases }, + dayOffsets: mergeRecord(base?.dayOffsets, overlay?.dayOffsets), + dayPartWords: mergeRecord(base?.dayPartWords, overlay?.dayPartWords), + dayTimePhrases: mergeRecord(base?.dayTimePhrases, overlay?.dayTimePhrases), + durationUnitSeconds: mergeRecord(base?.durationUnitSeconds, overlay?.durationUnitSeconds), + fillerWords: mergeList(base?.fillerWords ?? [], overlay?.fillerWords), + modifiers: mergeListRecord(base?.modifiers, overlay?.modifiers, DATE_MODIFIER_KEYS), + months: mergeRecord(base?.months, overlay?.months), + periodEdgePhrases: mergeRecord(base?.periodEdgePhrases, overlay?.periodEdgePhrases), + periodWords: mergeListRecord(base?.periodWords, overlay?.periodWords, DATE_PERIOD_KEYS), + relative: mergeListRecord(base?.relative, overlay?.relative, DATE_RELATIVE_KEYS), + subunit: mergeRecord(base?.subunit, overlay?.subunit), + timeAliases: mergeRecord(base?.timeAliases, overlay?.timeAliases), timeCorePattern: overlay?.timeCorePattern ?? base?.timeCorePattern, timePattern: overlay?.timePattern ?? base?.timePattern, - unitWords: { ...(base?.unitWords ?? {}), ...overlay?.unitWords }, + unitWords: mergeRecord(base?.unitWords, overlay?.unitWords), + weekdayOffsetPhrases: mergeRecord(base?.weekdayOffsetPhrases, overlay?.weekdayOffsetPhrases), weekdayNames: overlay?.weekdayNames ?? base?.weekdayNames ?? [], - weekdays: { ...(base?.weekdays ?? {}), ...overlay?.weekdays }, + weekdays: mergeRecord(base?.weekdays, overlay?.weekdays), } } +function mergeRecord(base?: Record, overlay?: Record): Record { + return { ...(base ?? {}), ...overlay } +} + +function mergeListRecord( + base: Partial> | undefined, + overlay: Partial> | undefined, + keys: readonly K[], +): Partial> | undefined { + if (!(base || overlay)) { + return + } + const result: Partial> = {} + for (const key of keys) { + result[key] = mergeList(base?.[key] ?? [], overlay?.[key]) + } + return result +} + function mergeSet(base: ReadonlySet, overlay?: WordSetInput): ReadonlySet { return new Set([...base, ...toWords(overlay)].map((word) => word.toLowerCase())) } @@ -328,6 +329,10 @@ function uniqueStrings(values: readonly string[]): readonly string[] { return [...new Set(values)] } +function sortLongestFirst(phrases: readonly string[]): readonly string[] { + return [...phrases].sort((a, b) => b.length - a.length || a.localeCompare(b)) +} + export function normalizeLocale(locale: string): string { return locale.toLowerCase().replace('_', '-') } diff --git a/packages/lingo/src/locale/types.ts b/packages/lingo/src/locale/types.ts index 4dc33b3..be1bc05 100644 --- a/packages/lingo/src/locale/types.ts +++ b/packages/lingo/src/locale/types.ts @@ -8,7 +8,7 @@ export type WordSetInput = ReadonlySet | readonly string[] export type DateOffsetUnit = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year' export type DateGrainUnit = 'hour' | 'minute' | 'second' export type DatePeriodUnit = 'week' | 'month' | 'year' -export type DateRelativeModifier = 'this' | 'next' | 'last' +export type DateRelativeModifier = 'this' | 'next' | 'last' | 'afterNext' | 'beforeLast' export interface GrammarBoundPhrase { bound: 'min' | 'max' @@ -17,6 +17,12 @@ export interface GrammarBoundPhrase { } export interface GrammarWords { + /** + * Multi-word leading approximate phrases matched longest-first before + * single-word approximateWords. E.g. "más o menos", "à peu près", + * "mais ou menos", "por volta de". Sorted longest-first at resolve time. + */ + approximatePhrases: readonly string[] approximateWords: ReadonlySet boundPhrases: readonly GrammarBoundPhrase[] compoundJoinWords: ReadonlySet @@ -59,6 +65,26 @@ export interface FuzzyAmountWord { export interface NumberWordTables { andWords: ReadonlySet articles: ReadonlySet + /** + * Scale words that may open a number expression without a preceding multiplier. + * E.g. Spanish "cien" (100), "mil" (1000); Portuguese "cem" (100). + * English "hundred"/"thousand" require "a hundred" — they stay out of this table. + */ + bareScales?: Record + /** + * Single- or multi-word exact number compounds resolved via longest-match + * before general word composition. Handles vigesimal French + * (quatre-vingts→80), Spanish hundreds (quinientos→500), etc. + * Multi-word entries are space-separated; matching uses the same folded text + * as other word tables. + */ + composed?: Record + /** + * Words meaning "decimal point" spoken between integer and fractional parts. + * E.g. Spanish "coma", French "virgule", Portuguese "vírgula", English "point". + * "dos coma cinco" → 2.5, "two point five" → 2.5. + */ + decimalWords?: ReadonlySet dozenWords: ReadonlySet fractionWords: Record fuzzyAmounts: Record @@ -73,7 +99,9 @@ export type NumberWordTablesInput = Omit< { [K in keyof NumberWordTables]: NumberWordTables[K] extends ReadonlySet ? WordSetInput - : NumberWordTables[K] + : NumberWordTables[K] extends ReadonlySet | undefined + ? WordSetInput + : NumberWordTables[K] }, never > @@ -115,6 +143,18 @@ export interface DateCalendarPeriodPhrase { period: DatePeriodUnit } +export interface DatePeriodEdgePhrase { + edge: 'start' | 'mid' | 'end' + /** + * Deterministic anchor for edge phrases. `start` resolves to the first day of + * the period, `mid` to the existing midpoint convention (week +3, month day + * 15, year July 2), and `end` to the last day of the period. Month-name + * composition is parser-owned (`fin juillet`); pack values do not carry month + * names. + */ + period: DatePeriodUnit +} + export interface DateCompactOffsetVocab { futureSuffixes?: readonly string[] pastSuffixes?: readonly string[] @@ -130,13 +170,18 @@ export interface DateRelativeVocab { export interface DateVocabPack { calendarPeriodPhrases?: Record + clockMinuteWords?: Record + clockPastWords?: readonly string[] + clockToWords?: readonly string[] compactOffset?: DateCompactOffsetVocab dayOffsets?: Record + dayPartWords?: Record dayTimePhrases?: Record durationUnitSeconds?: Record fillerWords?: readonly string[] modifiers?: Partial> months: Record + periodEdgePhrases?: Record periodWords?: Partial> relative?: DateRelativeVocab subunit: Partial> @@ -145,6 +190,7 @@ export interface DateVocabPack { timePattern?: string unitWords: Record weekdayNames: readonly string[] + weekdayOffsetPhrases?: Record weekdays: Record } diff --git a/packages/lingo/src/locales/es.ts b/packages/lingo/src/locales/es.ts index 11c317b..d0b81c4 100644 --- a/packages/lingo/src/locales/es.ts +++ b/packages/lingo/src/locales/es.ts @@ -4,6 +4,7 @@ export const es: LocalePack = { locale: 'es', aliases: ['es-ES', 'es-MX', 'es-AR', 'es-CL', 'es-CO'], extends: 'en', + detectionWords: ['ahorita'], defaults: { numberFormat: 'comma-decimal', }, @@ -40,7 +41,8 @@ export const es: LocalePack = { ], }, grammar: { - approximateWords: ['aproximadamente', 'aprox', 'alrededor', 'cerca', 'casi'], + approximatePhrases: ['mas o menos', 'por ahi de', 'alrededor de', 'cerca de', 'cosa de'], + approximateWords: ['aproximadamente', 'aprox', 'alrededor', 'cerca', 'casi', 'unos', 'unas'], boundPhrases: [ { phrase: 'al menos', bound: 'min', exclusive: false }, { phrase: 'como minimo', bound: 'min', exclusive: false }, @@ -54,14 +56,47 @@ export const es: LocalePack = { conversionWords: ['a', 'en', 'como'], exactWords: ['exactamente', 'precisamente'], phraseWords: ['entre', 'y', 'a', 'aproximadamente', 'alrededor', 'menos', 'mas', 'hasta'], + rangeAlternativeWords: ['o'], rangeAndWords: ['y'], rangeBetweenWords: ['entre'], - rangeFromWords: ['desde'], + rangeFromWords: ['desde', 'de'], rangeSeparatorWords: ['a', 'hasta'], + trailingApproxPhrases: ['y pico', 'y tantos', 'y algo', 'mas o menos'], }, numberWords: { andWords: ['y'], articles: ['un', 'una'], + bareScales: { cien: 100, ciento: 100, mil: 1000 }, + composed: { + veintiuno: 21, + veintiun: 21, + veintiuna: 21, + veintidos: 22, + veintitres: 23, + veinticuatro: 24, + veinticinco: 25, + veintiseis: 26, + veintisiete: 27, + veintiocho: 28, + veintinueve: 29, + doscientos: 200, + doscientas: 200, + trescientos: 300, + trescientas: 300, + cuatrocientos: 400, + cuatrocientas: 400, + quinientos: 500, + quinientas: 500, + seiscientos: 600, + seiscientas: 600, + setecientos: 700, + setecientas: 700, + ochocientos: 800, + ochocientas: 800, + novecientos: 900, + novecientas: 900, + }, + decimalWords: ['coma'], dozenWords: ['docena'], fractionWords: { medio: 1 / 2, @@ -72,6 +107,7 @@ export const es: LocalePack = { cuarta: 1 / 4, }, fuzzyAmounts: { + par: { value: 2, spread: [2, 3] }, varios: { value: 5, spread: [4, 7] }, algunas: { value: 3, spread: [2, 4] }, algunos: { value: 3, spread: [2, 4] }, @@ -122,20 +158,27 @@ export const es: LocalePack = { }, }, date: { + clockMinuteWords: { cuarto: 15, media: 30 }, + clockPastWords: ['y'], + clockToWords: ['menos'], dayOffsets: { hoy: 0, manana: 1, ayer: -1, + 'pasado manana': 2, + anteayer: -2, + antier: -2, }, dayTimePhrases: { 'en la manana': { hour: 9 }, 'por la manana': { hour: 9 }, + 'de madrugada': { hour: 4 }, 'manana por la manana': { dayOffset: 1, hour: 9 }, 'manana en la manana': { dayOffset: 1, hour: 9 }, 'mediodia manana': { dayOffset: 1, hour: 12 }, mediodia: { dayOffset: 0, hour: 12 }, }, - fillerWords: ['el', 'la', 'los', 'las', 'de', 'del'], + fillerWords: ['el', 'la', 'los', 'las', 'de', 'del', 'a'], modifiers: { this: ['este', 'esta'], next: ['proximo', 'proxima', 'que viene'], @@ -167,6 +210,11 @@ export const es: LocalePack = { diciembre: 11, dic: 11, }, + periodEdgePhrases: { + principios: { edge: 'start', period: 'month' }, + mediados: { edge: 'mid', period: 'month' }, + finales: { edge: 'end', period: 'month' }, + }, periodWords: { week: ['semana'], month: ['mes'], diff --git a/packages/lingo/src/locales/fr.ts b/packages/lingo/src/locales/fr.ts index 205a6a4..d5e6bae 100644 --- a/packages/lingo/src/locales/fr.ts +++ b/packages/lingo/src/locales/fr.ts @@ -40,6 +40,7 @@ export const fr: LocalePack = { ], }, grammar: { + approximatePhrases: ['a peu pres', 'plus ou moins', 'grosso modo'], approximateWords: ['environ', 'approximativement', 'approx', 'presque'], boundPhrases: [ { phrase: 'au moins', bound: 'min', exclusive: false }, @@ -55,14 +56,51 @@ export const fr: LocalePack = { conversionWords: ['en', 'vers', 'comme'], exactWords: ['exactement', 'precisement'], phraseWords: ['entre', 'et', 'en', 'environ', 'moins', 'plus', 'jusqu'], + rangeAlternativeWords: ['ou'], rangeAndWords: ['et'], rangeBetweenWords: ['entre'], rangeFromWords: ['de', 'depuis'], rangeSeparatorWords: ['a', 'en', 'jusqu'], + trailingApproxPhrases: ['et quelques', 'a peu pres'], + trailingApproxWords: ['environ'], }, numberWords: { andWords: ['et'], articles: ['un', 'une'], + bareScales: { cent: 100, mille: 1000 }, + composed: { + 'soixante dix': 70, + 'soixante et onze': 71, + 'soixante douze': 72, + 'soixante treize': 73, + 'soixante quatorze': 74, + 'soixante quinze': 75, + 'soixante seize': 76, + 'soixante dix sept': 77, + 'soixante dix huit': 78, + 'soixante dix neuf': 79, + 'quatre vingts': 80, + 'quatre vingt un': 81, + 'quatre vingt deux': 82, + 'quatre vingt trois': 83, + 'quatre vingt quatre': 84, + 'quatre vingt cinq': 85, + 'quatre vingt six': 86, + 'quatre vingt sept': 87, + 'quatre vingt huit': 88, + 'quatre vingt neuf': 89, + 'quatre vingt dix': 90, + 'quatre vingt onze': 91, + 'quatre vingt douze': 92, + 'quatre vingt treize': 93, + 'quatre vingt quatorze': 94, + 'quatre vingt quinze': 95, + 'quatre vingt seize': 96, + 'quatre vingt dix sept': 97, + 'quatre vingt dix huit': 98, + 'quatre vingt dix neuf': 99, + }, + decimalWords: ['virgule'], dozenWords: ['douzaine'], fractionWords: { demi: 1 / 2, @@ -73,6 +111,11 @@ export const fr: LocalePack = { quarts: 1 / 4, }, fuzzyAmounts: { + dizaine: { value: 10, spread: [8, 12] }, + vingtaine: { value: 20, spread: [18, 25] }, + trentaine: { value: 30, spread: [25, 35] }, + centaine: { value: 100, spread: [80, 120] }, + millier: { value: 1000, spread: [800, 1200] }, quelques: { value: 3, spread: [2, 4] }, plusieurs: { value: 5, spread: [4, 7] }, }, @@ -123,12 +166,22 @@ export const fr: LocalePack = { }, }, date: { + clockMinuteWords: { demi: 30, demie: 30, quart: 15 }, + clockPastWords: ['et'], + clockToWords: ['moins'], dayOffsets: { + "aujourd'hui": 0, demain: 1, hier: -1, + 'apres-demain': 2, + 'avant-hier': -2, }, dayTimePhrases: { 'ce matin': { hour: 9 }, + 'ce soir': { hour: 21 }, + 'cet apres-midi': { hour: 15 }, + 'hier soir': { dayOffset: -1, hour: 21 }, + 'demain soir': { dayOffset: 1, hour: 21 }, 'demain matin': { dayOffset: 1, hour: 9 }, 'midi demain': { dayOffset: 1, hour: 12 }, 'demain midi': { dayOffset: 1, hour: 12 }, @@ -165,6 +218,11 @@ export const fr: LocalePack = { decembre: 11, dec: 11, }, + periodEdgePhrases: { + debut: { edge: 'start', period: 'month' }, + mi: { edge: 'mid', period: 'month' }, + fin: { edge: 'end', period: 'month' }, + }, periodWords: { week: ['semaine'], month: ['mois'], @@ -196,6 +254,7 @@ export const fr: LocalePack = { annees: 'year', }, weekdayNames: ['dimanche', 'lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi'], + weekdayOffsetPhrases: { 'en huit': 7, 'en quinze': 14 }, weekdays: { dimanche: 0, dim: 0, diff --git a/packages/lingo/src/locales/ja.ts b/packages/lingo/src/locales/ja.ts index 01d59a5..9738743 100644 --- a/packages/lingo/src/locales/ja.ts +++ b/packages/lingo/src/locales/ja.ts @@ -8,6 +8,7 @@ export const ja: LocalePack = { numberFormat: 'auto', }, units: { + currency: [['JPY', '円']], length: [ ['m', 'メートル'], ['cm', 'センチ センチメートル'], @@ -62,9 +63,12 @@ export const ja: LocalePack = { rangeBetweenWords: ['間'], rangeFromWords: ['から'], rangeSeparatorWords: ['から', 'まで', 'へ', 'に'], + trailingApproxWords: ['ぐらい', 'くらい', 'ほど', '前後'], + trailingOkWords: ['個', '本', '匹'], }, numberWords: { andWords: ['と'], + decimalWords: ['点'], fractionWords: { 半: 1 / 2, }, @@ -89,14 +93,21 @@ export const ja: LocalePack = { 万: 10_000, 億: 100_000_000, }, + tens: { + 十: 10, + }, }, date: { calendarPeriodPhrases: { 来週: { modifier: 'next', period: 'week' }, 来月: { modifier: 'next', period: 'month' }, 来年: { modifier: 'next', period: 'year' }, + 再来週: { modifier: 'afterNext', period: 'week' }, + 再来月: { modifier: 'afterNext', period: 'month' }, 先週: { modifier: 'last', period: 'week' }, 先月: { modifier: 'last', period: 'month' }, + 先々週: { modifier: 'beforeLast', period: 'week' }, + 先々月: { modifier: 'beforeLast', period: 'month' }, 去年: { modifier: 'last', period: 'year' }, }, compactOffset: { @@ -118,12 +129,24 @@ export const ja: LocalePack = { dayOffsets: { 今日: 0, 明日: 1, + 明後日: 2, + あさって: 2, + しあさって: 3, 昨日: -1, + 一昨日: -2, + おととい: -2, }, dayTimePhrases: { + 今朝: { dayOffset: 0, hour: 8 }, + 今晩: { dayOffset: 0, hour: 19 }, + 今夜: { dayOffset: 0, hour: 21 }, 正午: { dayOffset: 0, hour: 12 }, 明日の正午: { dayOffset: 1, hour: 12 }, }, + periodEdgePhrases: { + 月末: { edge: 'end', period: 'month' }, + 週末: { edge: 'end', period: 'week' }, + }, timeAliases: { 正午: { hour: 12 }, 真夜中: { hour: 0 }, diff --git a/packages/lingo/src/locales/pt.ts b/packages/lingo/src/locales/pt.ts index 3a1d2de..5349b95 100644 --- a/packages/lingo/src/locales/pt.ts +++ b/packages/lingo/src/locales/pt.ts @@ -40,7 +40,8 @@ export const pt: LocalePack = { ], }, grammar: { - approximateWords: ['aproximadamente', 'aprox', 'cerca', 'quase'], + approximatePhrases: ['mais ou menos', 'cerca de', 'por volta de', 'la pelas'], + approximateWords: ['aproximadamente', 'aprox', 'cerca', 'quase', 'uns', 'umas'], boundPhrases: [ { phrase: 'pelo menos', bound: 'min', exclusive: false }, { phrase: 'no minimo', bound: 'min', exclusive: false }, @@ -54,14 +55,36 @@ export const pt: LocalePack = { conversionWords: ['para', 'em', 'como'], exactWords: ['exatamente', 'precisamente'], phraseWords: ['entre', 'e', 'para', 'aproximadamente', 'cerca', 'menos', 'mais', 'ate'], + rangeAlternativeWords: ['ou'], rangeAndWords: ['e'], rangeBetweenWords: ['entre'], rangeFromWords: ['de', 'desde'], rangeSeparatorWords: ['a', 'ate', 'para'], + trailingApproxPhrases: ['mais ou menos', 'e pouco', 'e poucos', 'e tanto'], }, numberWords: { andWords: ['e'], articles: ['um', 'uma'], + bareScales: { cem: 100, mil: 1000 }, + composed: { + duzentos: 200, + duzentas: 200, + trezentos: 300, + trezentas: 300, + quatrocentos: 400, + quatrocentas: 400, + quinhentos: 500, + quinhentas: 500, + seiscentos: 600, + seiscentas: 600, + setecentos: 700, + setecentas: 700, + oitocentos: 800, + oitocentas: 800, + novecentos: 900, + novecentas: 900, + }, + decimalWords: ['virgula'], dozenWords: ['duzia'], fractionWords: { meio: 1 / 2, @@ -72,6 +95,7 @@ export const pt: LocalePack = { quarta: 1 / 4, }, fuzzyAmounts: { + par: { value: 2, spread: [2, 3] }, alguns: { value: 3, spread: [2, 4] }, algumas: { value: 3, spread: [2, 4] }, varios: { value: 5, spread: [4, 7] }, @@ -100,9 +124,12 @@ export const pt: LocalePack = { catorze: 14, quinze: 15, dezesseis: 16, + dezasseis: 16, dezassete: 17, + dezessete: 17, dezoito: 18, dezenove: 19, + dezanove: 19, }, scales: { cem: 100, @@ -125,13 +152,19 @@ export const pt: LocalePack = { }, }, date: { + clockMinuteWords: { quarto: 15, meia: 30 }, + clockPastWords: ['e'], + clockToWords: ['para'], dayOffsets: { hoje: 0, amanha: 1, ontem: -1, + anteontem: -2, + 'depois de amanha': 2, }, dayTimePhrases: { 'de manha': { hour: 9 }, + 'de madrugada': { hour: 4 }, 'amanha de manha': { dayOffset: 1, hour: 9 }, 'meio-dia amanha': { dayOffset: 1, hour: 12 }, 'amanha meio-dia': { dayOffset: 1, hour: 12 }, @@ -140,7 +173,7 @@ export const pt: LocalePack = { 'meio-dia': { dayOffset: 0, hour: 12 }, 'meio dia': { dayOffset: 0, hour: 12 }, }, - fillerWords: ['o', 'a', 'os', 'as', 'de', 'do', 'da'], + fillerWords: ['o', 'a', 'os', 'as', 'de', 'do', 'da', 'no'], modifiers: { this: ['este', 'esta'], next: ['proximo', 'proxima', 'que vem'], @@ -171,6 +204,13 @@ export const pt: LocalePack = { dezembro: 11, dez: 11, }, + periodEdgePhrases: { + inicio: { edge: 'start', period: 'month' }, + comeco: { edge: 'start', period: 'month' }, + meio: { edge: 'mid', period: 'month' }, + fim: { edge: 'end', period: 'month' }, + final: { edge: 'end', period: 'month' }, + }, periodWords: { week: ['semana'], month: ['mes'], diff --git a/packages/lingo/src/locales/zh.ts b/packages/lingo/src/locales/zh.ts index 72e4626..7542157 100644 --- a/packages/lingo/src/locales/zh.ts +++ b/packages/lingo/src/locales/zh.ts @@ -33,10 +33,10 @@ export const zh: LocalePack = { duration: [ ['s', '秒'], ['min', '分钟'], - ['h', '小时'], + ['h', '小时 个小时'], ['d', '天'], ['wk', '周 星期'], - ['mo', '月'], + ['mo', '月 个月'], ['yr', '年'], ], }, @@ -47,7 +47,7 @@ export const zh: LocalePack = { }, ], grammar: { - approximateWords: ['约', '大约', '大概', '左右', '很热'], + approximateWords: ['约', '大约', '大概', '差不多', '左右', '很热'], boundPhrases: [ { phrase: '至少', bound: 'min', exclusive: false }, { phrase: '不少于', bound: 'min', exclusive: false }, @@ -58,13 +58,17 @@ export const zh: LocalePack = { ], conversionWords: ['到', '为', '成'], phraseWords: ['从', '到', '至', '约', '大约', '至少', '最多'], + rangeAlternativeWords: ['或'], rangeAndWords: ['和', '与'], rangeBetweenWords: ['介于'], rangeFromWords: ['从'], rangeSeparatorWords: ['到', '至'], + trailingApproxWords: ['左右', '上下'], + trailingOkWords: ['个'], }, numberWords: { andWords: ['和'], + decimalWords: ['点'], fractionWords: { 半: 1 / 2, }, @@ -73,6 +77,7 @@ export const zh: LocalePack = { 零: 0, 〇: 0, 一: 1, + 幺: 1, 二: 2, 两: 2, 三: 3, @@ -90,6 +95,9 @@ export const zh: LocalePack = { 万: 10_000, 亿: 100_000_000, }, + tens: { + 十: 10, + }, }, date: { calendarPeriodPhrases: { @@ -101,12 +109,13 @@ export const zh: LocalePack = { 去年: { modifier: 'last', period: 'year' }, }, compactOffset: { - futureSuffixes: ['后'], + futureSuffixes: ['后', '以后'], pastSuffixes: ['前'], unitWords: { 秒: 'second', 分钟: 'minute', 分: 'minute', + 个小时: 'hour', 小时: 'hour', 天: 'day', 日: 'day', @@ -120,12 +129,24 @@ export const zh: LocalePack = { dayOffsets: { 今天: 0, 明天: 1, + 后天: 2, + 大后天: 3, 昨天: -1, + 前天: -2, + 大前天: -3, }, dayTimePhrases: { 中午: { dayOffset: 0, hour: 12 }, 明天中午: { dayOffset: 1, hour: 12 }, }, + periodEdgePhrases: { + 月初: { edge: 'start', period: 'month' }, + 月中: { edge: 'mid', period: 'month' }, + 月底: { edge: 'end', period: 'month' }, + 年初: { edge: 'start', period: 'year' }, + 年底: { edge: 'end', period: 'year' }, + 周末: { edge: 'end', period: 'week' }, + }, timeAliases: { 中午: { hour: 12 }, 午夜: { hour: 0 }, @@ -135,6 +156,7 @@ export const zh: LocalePack = { 零: 0, 〇: 0, 一: 1, + 幺: 1, 二: 2, 两: 2, 三: 3, diff --git a/packages/lingo/src/number/cjk.test.ts b/packages/lingo/src/number/cjk.test.ts new file mode 100644 index 0000000..639c6c6 --- /dev/null +++ b/packages/lingo/src/number/cjk.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest' +import { resolveLanguageProfile } from '../locale/profile' +import { ja } from '../locales/ja' +import { zh } from '../locales/zh' +import { parseCjkNumberText } from './cjk' + +const zhNumbers = resolveLanguageProfile([zh], 'zh').numberWords +const jaNumbers = resolveLanguageProfile([ja], 'ja').numberWords + +function value(text: string, locale: 'zh' | 'ja' = 'zh'): number | null { + return parseCjkNumberText(text, locale === 'zh' ? zhNumbers : jaNumbers)?.value ?? null +} + +describe('CJK number algebra', () => { + it('composes positional and grouped Chinese numerals', () => { + expect(value('十五')).toBe(15) + expect(value('一百五十')).toBe(150) + expect(value('三百五十万')).toBe(3_500_000) + }) + + it('handles Chinese elliptical shorthands', () => { + expect(value('一百五')).toBe(150) + expect(value('三万五')).toBe(35_000) + expect(value('两千五')).toBe(2500) + }) + + it('composes mixed Arabic digits with CJK scales', () => { + expect(value('3万')).toBe(30_000) + expect(value('35万')).toBe(350_000) + expect(value('3万5千')).toBe(35_000) + expect(value('1億2千万', 'ja')).toBe(120_000_000) + }) + + it('rejects adjacent bare large scales without breaking valid groups', () => { + expect(parseCjkNumberText('万万', zhNumbers)).toBeNull() + expect(parseCjkNumberText('亿万', zhNumbers)).toBeNull() + expect(value('千万')).toBe(10_000_000) + expect(value('一億二千万', 'ja')).toBe(120_000_000) + }) + + it('detects only consecutive increasing one-character adjacent ranges', () => { + const range = parseCjkNumberText('三四个', zhNumbers) + expect(range).toMatchObject({ value: 3, end: 1, adjacentRange: true }) + expect(parseCjkNumberText('三三', zhNumbers)).toBeNull() + expect(value('十五')).toBe(15) + }) +}) diff --git a/packages/lingo/src/number/cjk.ts b/packages/lingo/src/number/cjk.ts new file mode 100644 index 0000000..fa3a4d0 --- /dev/null +++ b/packages/lingo/src/number/cjk.ts @@ -0,0 +1,246 @@ +import type { NumberWordTables } from '../locale/types' +import type { Token } from '../parse/tokenize' + +export interface CjkNumberResult { + adjacentRange?: boolean + end: number + sawScale: boolean + value: number +} + +export function parseCjkNumberText(text: string, tables: NumberWordTables): CjkNumberResult | null { + if (!hasCjk(text)) { + return null + } + let pos = 0 + let total = 0 + let group = 0 + let pending: number | null = null + let lastSmall = 0 + let lastGroup = 0 + let zeroBreak = false + let consumed = false + let sawScale = false + let bare: { end: number; value: number } | null = null + let largeScaleWithoutInterveningDigit = false + + while (pos < text.length) { + const digits = /^[0-9]+/.exec(text.slice(pos)) + if (digits) { + pending = Number(digits[0]) + bare = null + largeScaleWithoutInterveningDigit = false + consumed = true + pos += digits[0].length + continue + } + + const ch = text[pos]! + if (!hasCjk(ch)) { + break + } + const scale = tables.scales[ch] + if (scale !== undefined) { + if (scale >= 10_000) { + if (largeScaleWithoutInterveningDigit) { + return null + } + const value = finishGroup(group, pending, lastSmall, total, lastGroup, zeroBreak) + total += (value === 0 ? 1 : value) * scale + group = 0 + pending = null + lastSmall = 0 + lastGroup = scale + largeScaleWithoutInterveningDigit = true + } else { + group += (pending ?? 1) * scale + pending = null + lastSmall = scale + } + zeroBreak = false + bare = null + consumed = true + sawScale = true + pos++ + continue + } + + const one = tables.ones[ch] + if (one === 10) { + group += (pending ?? 1) * 10 + pending = null + lastSmall = 10 + zeroBreak = false + bare = null + consumed = true + sawScale = true + pos++ + continue + } + if (one === undefined || one < 0 || one > 9) { + break + } + if (bare && group === 0 && total === 0 && lastSmall === 0) { + return one === bare.value + 1 && one <= 9 + ? { value: bare.value, end: bare.end, sawScale: false, adjacentRange: true } + : null + } + if (one === 0) { + pending = 0 + zeroBreak = true + bare = null + } else if (pending === 0 && zeroBreak) { + pending = one + } else if (pending !== null && group === 0 && total === 0 && lastSmall === 0) { + return null + } else { + pending = one + zeroBreak = false + bare = lastSmall === 0 && total === 0 && group === 0 ? { value: one, end: pos + 1 } : null + } + largeScaleWithoutInterveningDigit = false + consumed = true + pos++ + } + + return consumed + ? { + value: total + finishGroup(group, pending, lastSmall, total, lastGroup, zeroBreak), + end: pos, + sawScale, + } + : null +} + +export function prepareCjkValueTokens(tokens: Token[], i: number, tables: NumberWordTables): void { + const t = tokens[i] + if (!t) { + return + } + if (t.type === 'word') { + const parsed = parseCjkNumberText(t.text, tables) + if (parsed && parsed.end > 0) { + splitPrefix(tokens, i, t.start + parsed.end, String(parsed.value), tables) + } + return + } + if (t.type !== 'digits') { + return + } + splitHalfSuffix(tokens, i + 1, tables) + const next = tokens[i + 1] + if (next?.type !== 'word' || next.spaceBefore) { + return + } + const parsed = parseCjkNumberText(contiguousText(tokens, i), tables) + if (parsed?.sawScale && parsed.end > t.text.length) { + splitPrefix(tokens, i, t.start + parsed.end, String(parsed.value), tables) + } +} + +export function consumeCjkPostUnitHalf( + tokens: Token[], + i: number, + tables: NumberWordTables, +): { end: number; next: number } | null { + const t = tokens[i] + return t && (t.type === 'word' || t.type === 'sym') && isHalf(t.text, tables) + ? { end: t.end, next: i + 1 } + : null +} + +function finishGroup( + group: number, + pending: number | null, + lastSmall: number, + total: number, + lastGroup: number, + zeroBreak: boolean, +): number { + if (pending === null) { + return group + } + if (!zeroBreak && pending > 0 && pending < 10 && lastSmall > 10) { + return group + pending * (lastSmall / 10) + } + if ( + !zeroBreak && + pending > 0 && + pending < 10 && + group === 0 && + total > 0 && + lastGroup >= 10_000 + ) { + return pending * (lastGroup / 10) + } + return group + pending +} + +function splitPrefix( + tokens: Token[], + i: number, + end: number, + value: string, + tables: NumberWordTables, +): void { + const first = tokens[i] + if (!first || end <= first.start) { + return + } + let cursor = i + while (tokens[cursor] && tokens[cursor]!.start < end) { + cursor++ + } + const pieces: Token[] = [ + { type: 'digits', text: value, start: first.start, end, spaceBefore: first.spaceBefore }, + ] + const last = tokens[cursor - 1] + if (last && end < last.end) { + pushWord(pieces, last.text.slice(end - last.start), end, last.end, tables) + } + tokens.splice(i, cursor - i, ...pieces) +} + +function splitHalfSuffix(tokens: Token[], i: number, tables: NumberWordTables): void { + const t = tokens[i] + if (t?.type !== 'word' || t.spaceBefore) { + return + } + const pieces: Token[] = [] + pushWord(pieces, t.text, t.start, t.end, tables) + if (pieces.length > 1) { + tokens.splice(i, 1, ...pieces) + } +} + +function pushWord( + out: Token[], + text: string, + start: number, + end: number, + tables: NumberWordTables, +): void { + const half = text[text.length - 1]! + if (text.length > 1 && isHalf(half, tables)) { + out.push({ type: 'word', text: text.slice(0, -1), start, end: end - 1, spaceBefore: false }) + out.push({ type: 'sym', text: half, start: end - 1, end, spaceBefore: false }) + } else { + out.push({ type: 'word', text, start, end, spaceBefore: false }) + } +} + +function contiguousText(tokens: Token[], i: number): string { + let text = '' + for (let pos = i; tokens[pos] && (pos === i || !tokens[pos]!.spaceBefore); pos++) { + text += tokens[pos]!.text + } + return text +} + +function isHalf(text: string, tables: NumberWordTables): boolean { + return hasCjk(text) && tables.fractionWords[text] === 1 / 2 +} + +function hasCjk(text: string): boolean { + return /[\u3400-\u4dbf\u4e00-\u9fff\u3040-\u30ff\uf900-\ufaff]/.test(text) +} diff --git a/packages/lingo/src/number/words-romance.test.ts b/packages/lingo/src/number/words-romance.test.ts new file mode 100644 index 0000000..c58fa95 --- /dev/null +++ b/packages/lingo/src/number/words-romance.test.ts @@ -0,0 +1,622 @@ +import { describe, expect, it } from 'vitest' +import { createLingo } from '../index' +import type { LocalePack } from '../locale/types' +import { es } from '../locales/es' +import { fr } from '../locales/fr' +import { pt } from '../locales/pt' + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +function makePack(base: LocalePack, overrides: Partial): LocalePack { + return { + ...base, + ...overrides, + grammar: { ...(base.grammar ?? {}), ...(overrides.grammar ?? {}) }, + numberWords: { ...(base.numberWords ?? {}), ...(overrides.numberWords ?? {}) }, + } as LocalePack +} + +function qty(input: string, pack: LocalePack, locale?: string) { + const lingo = createLingo({ locales: [pack] }) + return lingo.parseQuantity(input, { locale: locale ?? pack.locale }) +} + +function range(input: string, pack: LocalePack, locale?: string) { + const lingo = createLingo({ locales: [pack] }) + return lingo.parseRange(input, { locale: locale ?? pack.locale }) +} + +// ─── Feature 1: tens + and-word + ones composition ────────────────────────── + +describe('tens + and-word + ones composition', () => { + it('parses Spanish treinta y cinco = 35', () => { + const r = qty('treinta y cinco kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(35) + } + }) + + it('parses Spanish cuarenta y dos = 42', () => { + const r = qty('cuarenta y dos kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(42) + } + }) + + it('parses Spanish noventa y nueve = 99', () => { + const r = qty('noventa y nueve kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(99) + } + }) + + it('parses Portuguese vinte e cinco = 25', () => { + const r = qty('vinte e cinco kg', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(25) + } + }) + + it('parses Portuguese trinta e sete = 37', () => { + const r = qty('trinta e sete kg', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(37) + } + }) + + it('parses French vingt et un = 21', () => { + const r = qty('vingt et un kg', fr) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(21) + } + }) + + it('parses French trente et un = 31', () => { + const r = qty('trente et un kg', fr) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(31) + } + }) + + // GUARD: and-word in range context must NOT be consumed as number composition + it('preserves "entre cinco y diez kilos" as range', () => { + const r = range('entre cinco y diez kilos', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(5) + expect(r.range.max()?.value).toBe(10) + } + }) + + it('preserves "entre 5 y 10 kg" as range', () => { + const r = range('entre 5 y 10 kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(5) + expect(r.range.max()?.value).toBe(10) + } + }) + + it('preserves "entre 5 e 10 kg" as range in Portuguese', () => { + const r = range('entre 5 e 10 kg', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(5) + expect(r.range.max()?.value).toBe(10) + } + }) + + it('does not affect English (no and-word composition for English tens)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('twenty five kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(25) // hyphen/space form still works + } + // "twenty and five" in English: "and" is in andWords so it composes + // into tens+ones (same mechanism as romance languages). This is NOT + // the same as scale-and ("one hundred and five") — it's handled by + // the hyphen/and-word skip after tens. + const r2 = lingo.parseQuantity('twenty and five kg') + expect(r2.ok).toBe(true) + if (r2.ok) { + expect(r2.quantity.value).toBe(25) + } + }) +}) + +// ─── Feature 2: Bare scale words ──────────────────────────────────────────── + +describe('bare scale words', () => { + const esBare = makePack(es, { + numberWords: { ...es.numberWords, bareScales: { cien: 100, ciento: 100, mil: 1000 } }, + }) + + const ptBare = makePack(pt, { + numberWords: { ...pt.numberWords, bareScales: { cem: 100, mil: 1000 } }, + }) + + it('parses "cien gramos" = 100 g', () => { + const r = qty('cien gramos', esBare) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + expect(r.quantity.unit).toBe('g') + } + }) + + it('parses "mil metros" = 1000 m', () => { + const r = qty('mil metros', esBare) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1000) + expect(r.quantity.unit).toBe('m') + } + }) + + it('parses "cem quilos" = 100 kg', () => { + const r = qty('cem quilos', ptBare) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + expect(r.quantity.unit).toBe('kg') + } + }) + + it('English "hundred" does NOT open bare (requires "a hundred")', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('hundred kg') + expect(r.ok).toBe(false) + }) + + it('English "a hundred" still works', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('a hundred kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + } + }) + + it('bare scale continues composing: "ciento cincuenta gramos" = 150', () => { + const r = qty('ciento cincuenta gramos', esBare) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(150) + } + }) +}) + +// ─── Feature 3: composed table ────────────────────────────────────────────── + +describe('composed table (vigesimal, compound hundreds)', () => { + const frComposed = makePack(fr, { + numberWords: { + ...fr.numberWords, + composed: { + 'quatre vingts': 80, + 'quatre vingt dix': 90, + 'quatre vingt dix neuf': 99, + 'soixante dix': 70, + 'soixante quinze': 75, + }, + }, + }) + + const esComposed = makePack(es, { + numberWords: { + ...es.numberWords, + bareScales: { cien: 100, ciento: 100, mil: 1000 }, + composed: { quinientos: 500, doscientos: 200, trescientos: 300 }, + }, + }) + + it('parses "quatre-vingts kg" = 80', () => { + const r = qty('quatre-vingts kg', frComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(80) + } + }) + + it('parses "quatre-vingt-dix kg" = 90', () => { + const r = qty('quatre-vingt-dix kg', frComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(90) + } + }) + + it('parses "quatre-vingt-dix-neuf kg" = 99', () => { + const r = qty('quatre-vingt-dix-neuf kg', frComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(99) + } + }) + + it('parses "soixante-dix kg" = 70', () => { + const r = qty('soixante-dix kg', frComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(70) + } + }) + + it('parses "soixante-quinze kg" = 75', () => { + const r = qty('soixante-quinze kg', frComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(75) + } + }) + + it('parses "quinientos kg" = 500', () => { + const r = qty('quinientos kg', esComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(500) + } + }) + + it('parses "doscientos kg" = 200', () => { + const r = qty('doscientos kg', esComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(200) + } + }) + + it('composed + scale: "mil quinientos kg" = 1500', () => { + const r = qty('mil quinientos kg', esComposed) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1500) + } + }) + + it('scale + composed after and: "mille quatre-vingts kg" = 1080', () => { + const frScale = makePack(fr, { + numberWords: { + ...fr.numberWords, + bareScales: { mille: 1000 }, + composed: { 'quatre vingts': 80 }, + }, + }) + const r = qty('mille quatre-vingts kg', frScale) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1080) + } + }) +}) + +// ─── Feature 4: spoken decimal separator words ────────────────────────────── + +describe('spoken decimal separator words', () => { + it('parses English "two point five kg" = 2.5', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('two point five kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) + + it('parses English "three point fourteen kg" = 3.14', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('three point fourteen kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBeCloseTo(3.14, 10) + } + }) + + it('parses Spanish "dos coma cinco kg" = 2.5', () => { + const esDec = makePack(es, { + numberWords: { ...es.numberWords, decimalWords: ['coma'] }, + }) + const r = qty('dos coma cinco kg', esDec) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) + + it('parses French "trois virgule quatorze kg" = 3.14', () => { + const frDec = makePack(fr, { + numberWords: { ...fr.numberWords, decimalWords: ['virgule'] }, + }) + const r = qty('trois virgule quatorze kg', frDec) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBeCloseTo(3.14, 10) + } + }) + + it('parses Portuguese "dois virgula cinco kg" = 2.5', () => { + const ptDec = makePack(pt, { + numberWords: { ...pt.numberWords, decimalWords: ['virgula'] }, + }) + const r = qty('dois virgula cinco kg', ptDec) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) + + it('decimal word followed by digits: "two point 5 kg" = 2.5', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('two point 5 kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) +}) + +// ─── Feature 5: leading multi-word approximant phrases ────────────────────── + +describe('leading multi-word approximant phrases', () => { + const esApprox = makePack(es, { + grammar: { ...es.grammar, approximatePhrases: ['mas o menos'] }, + }) + + const ptApprox = makePack(pt, { + grammar: { ...pt.grammar, approximatePhrases: ['mais ou menos', 'por volta de'] }, + }) + + const frApprox = makePack(fr, { + grammar: { ...fr.grammar, approximatePhrases: ['a peu pres'] }, + }) + + it('parses "mas o menos 5 kilos" as approximate', () => { + const r = qty('mas o menos 5 kilos', esApprox) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(5) + expect(r.quantity.approximate).toBe(true) + } + }) + + it('parses "mais ou menos 5 quilos" as approximate', () => { + const r = qty('mais ou menos 5 quilos', ptApprox) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(5) + expect(r.quantity.approximate).toBe(true) + } + }) + + it('parses "por volta de 3 litros" as approximate', () => { + const r = qty('por volta de 3 litros', ptApprox) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(3) + expect(r.quantity.approximate).toBe(true) + } + }) + + it('parses "a peu pres 5 kg" as approximate', () => { + const r = qty('a peu pres 5 kg', frApprox) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(5) + expect(r.quantity.approximate).toBe(true) + } + }) + + it('English has no approximatePhrases by default (empty)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('about 5 kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(5) + expect(r.quantity.approximate).toBe(true) + } + }) + + it('approx phrase composes with "A a B" separator range (range parses)', () => { + const r = range('mas o menos 5 a 10 kg', esApprox) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(5) + expect(r.range.max()?.value).toBe(10) + // Note: approximate propagation to separator-style ranges is a known + // architectural limit (same in English "about 5 to 10 kg") — the range + // still parses successfully with the phrase consumed as a qualifier. + } + }) +}) + +// ─── Regression: English unchanged ────────────────────────────────────────── + +describe('English behavior unchanged', () => { + it('"a hundred" still parses', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('a hundred kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + } + }) + + it('"1,500 kg" still parses', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('1,500 kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1500) + } + }) + + it('"twenty-five kg" still works', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('twenty-five kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(25) + } + }) + + it('"one hundred and five kg" still works', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('one hundred and five kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(105) + } + }) + + it('"between 5 and 10 kg" still parses as range', () => { + const lingo = createLingo() + const r = lingo.parseRange('between 5 and 10 kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(5) + expect(r.range.max()?.value).toBe(10) + } + }) +}) + +// ─── F1: spoken decimal tails as digit sequence ───────────────────────────── + +describe('F1: spoken decimal tails compose as digit sequence', () => { + it('"dos coma cinco seis kg" = 2.56 (not 2.11)', () => { + const r = qty('dos coma cinco seis kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.56) + } + }) + + it('"quinientos coma cero cinco kg" = 500.05 (leading zero preserved)', () => { + const r = qty('quinientos coma cero cinco kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(500.05) + } + }) + + it('"two point five six kg" = 2.56 (English)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('two point five six kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.56) + } + }) + + it('"trois virgule quatorze kg" = 3.14 (regression: multi-digit group)', () => { + const r = qty('trois virgule quatorze kg', fr) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBeCloseTo(3.14, 10) + } + }) + + it('"two point five kg" = 2.5 (single digit still works)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('two point five kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) +}) + +// ─── F2: article-led scale chains ─────────────────────────────────────────── + +describe('F2: article-led scale chains continue composing', () => { + it('"un millon quinientos mil kg" = 1,500,000 (ES)', () => { + const r = qty('un millon quinientos mil kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1_500_000) + } + }) + + it('"um milhao quinhentos mil kg" = 1,500,000 (PT)', () => { + const r = qty('um milhao quinhentos mil kg', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1_500_000) + } + }) + + it('"un million cinq cent mille kg" = 1,500,000 (FR)', () => { + const r = qty('un million cinq cent mille kg', fr) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1_500_000) + } + }) + + it('"a hundred kg" (en) still parses as 100', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('a hundred kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + } + }) + + it('"un kg" (es) still parses as 1', () => { + const r = qty('un kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1) + } + }) +}) + +// ─── F4: feminine veintiuna ───────────────────────────────────────────────── + +describe('F4: feminine veintiuna', () => { + it('"veintiuna pulgadas" = 21 in', () => { + const r = qty('veintiuna pulgadas', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(21) + expect(r.quantity.unit).toBe('in') + } + }) +}) + +// ─── F5: longest-first phrase sorting + trailing phrases ──────────────────── + +describe('F5: approximatePhrases longest-first + trailing variants', () => { + it('"mas o menos 5 kg" parses as approx (not partial match on "mas")', () => { + const r = qty('mas o menos 5 kg', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(5) + expect(r.quantity.approximate).toBe(true) + } + }) + + it('"5 kg mas o menos" trailing approx (ES)', () => { + const r = qty('5 kg mas o menos', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(5) + expect(r.quantity.approximate).toBe(true) + } + }) + + it('"5 quilos mais ou menos" trailing approx (PT)', () => { + const r = qty('5 quilos mais ou menos', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(5) + expect(r.quantity.approximate).toBe(true) + } + }) +}) diff --git a/packages/lingo/src/number/words.ts b/packages/lingo/src/number/words.ts index 0f2a810..a6376e9 100644 --- a/packages/lingo/src/number/words.ts +++ b/packages/lingo/src/number/words.ts @@ -64,7 +64,7 @@ export function parseNumberWords( } function parseCore(tokens: Token[], i: number, tables: NumberWordTables): WordNumberResult | null { - let pos = i + const pos = i const w = word(tokens[pos]) if (w === null) { return null @@ -129,58 +129,131 @@ function parseCore(tokens: Token[], i: number, tables: NumberWordTables): WordNu } } + // Composed table: longest-match phrase lookup (quinientos, quatre-vingt-dix, etc.) + const comp = tryComposed(tokens, pos, tables) + if (comp) { + return cardinalLoop(tokens, comp.next, 0, comp.value, true, tables) + } + + // Bare scale words: "cien gramos", "mil metros" + if (tables.bareScales?.[w] !== undefined) { + return cardinalLoop(tokens, pos + 1, 0, tables.bareScales[w]!, true, tables) + } + // Standard cardinal grammar. - let total = 0 - let current = 0 - let consumedAny = false - let sawDozen = false + return cardinalLoop(tokens, pos, 0, 0, false, tables) +} + +/** Longest-match against composed table; hyphens transparent. */ +function tryComposed( + tokens: Token[], + pos: number, + tables: NumberWordTables, +): { next: number; value: number } | null { + const c = tables.composed + if (!c) { + return null + } + let best: number | undefined + let bestNext = pos + let key = '' + let scan = pos + let end = pos + while (scan < tokens.length) { + const t = tokens[scan]! + if (t.type === 'word') { + key = key ? `${key} ${t.text.toLowerCase()}` : t.text.toLowerCase() + end = ++scan + } else if (t.type === 'sym' && t.text === '-' && word(tokens[scan + 1]) !== null) { + scan++ + continue + } else { + break + } + if (c[key] !== undefined) { + best = c[key]! + bestNext = end + } + } + return best === undefined ? null : { value: best, next: bestNext } +} + +function cardinalLoop( + tokens: Token[], + startPos: number, + total: number, + current: number, + any: boolean, + tables: NumberWordTables, +): WordNumberResult | null { + let pos = startPos + let t_ = total + let c_ = current + let dozen = false while (pos < tokens.length) { - const t = tokens[pos]! - const tw = word(t) + const tw = word(tokens[pos]) if (tw === null) { break } + + if (any && tables.composed) { + const m = tryComposed(tokens, pos, tables) + if (m) { + c_ += m.value + pos = m.next + continue + } + } if (tables.ones[tw] !== undefined) { - current += tables.ones[tw]! + c_ += tables.ones[tw]! pos++ - consumedAny = true - // hyphenated tens handled below via TENS branch; ones ends a group + any = true continue } if (tables.tens[tw] !== undefined) { - current += tables.tens[tw]! + c_ += tables.tens[tw]! pos++ - consumedAny = true - // optional hyphen + ones ("twenty-five" / "twenty five") - const hyphen = tokens[pos] - if (hyphen && hyphen.type === 'sym' && hyphen.text === '-') { - const onesW = word(tokens[pos + 1]) - if (onesW && tables.ones[onesW] !== undefined && tables.ones[onesW]! < 10) { - current += tables.ones[onesW]! - pos += 2 + any = true + // hyphen+ones OR and-word+ones after tens + const nxt = tokens[pos] + const skip = + nxt && nxt.type === 'sym' && nxt.text === '-' + ? 1 + : word(tokens[pos]) && tables.andWords.has(word(tokens[pos])!) + ? 1 + : 0 + if (skip) { + const o = word(tokens[pos + skip]) + if (o && tables.ones[o] !== undefined && tables.ones[o]! < 10) { + c_ += tables.ones[o]! + pos += skip + 1 } } continue } - if (tables.dozenWords.has(tw) && consumedAny) { - current *= 12 - sawDozen = true + if (tables.dozenWords.has(tw) && any) { + c_ *= 12 + dozen = true pos++ continue } - if (tables.scales[tw] !== undefined && consumedAny) { + if (tables.scales[tw] !== undefined && any) { if (tables.scales[tw] === 100) { - current = (current === 0 ? 1 : current) * 100 + c_ = (c_ === 0 ? 1 : c_) * 100 } else { - total += (current === 0 ? 1 : current) * tables.scales[tw]! - current = 0 + t_ += (c_ === 0 ? 1 : c_) * tables.scales[tw]! + c_ = 0 } pos++ - // optional "and" ("one hundred and five") - const join = word(tokens[pos]) - if (join && tables.andWords.has(join)) { - const after = word(tokens[pos + 1]) - if (after && (tables.ones[after] !== undefined || tables.tens[after] !== undefined)) { + const j = word(tokens[pos]) + if (j && tables.andWords.has(j)) { + const a = word(tokens[pos + 1]) + if ( + a && + (tables.ones[a] !== undefined || + tables.tens[a] !== undefined || + tables.composed?.[a] !== undefined) + ) { pos++ } } @@ -188,14 +261,53 @@ function parseCore(tokens: Token[], i: number, tables: NumberWordTables): WordNu } break } - if (!consumedAny) { + if (!any) { return null } - let value = total + current - if (sawDozen) { + let value = t_ + c_ + + // Spoken decimal separator — parse post-decimal as digit sequence + if (tables.decimalWords) { + const dw = word(tokens[pos]) + if (dw && tables.decimalWords.has(dw)) { + let d = '' + let dp = pos + 1 + while (dp < tokens.length) { + const tw = word(tokens[dp]) + if (!tw) { + break + } + const v = tables.ones[tw] + if (v !== undefined && v <= 99) { + d += String(v) + dp++ + } else if (tables.composed) { + const c = tryComposed(tokens, dp, tables) + if (c && c.value <= 99) { + d += String(c.value) + dp = c.next + } else { + break + } + } else { + break + } + } + if (!d && tokens[pos + 1]?.type === 'digits') { + const tx = tokens[pos + 1]!.text + d = tx + dp = pos + 2 + } + if (d) { + const frac = +d / 10 ** d.length + return { value: value + (value < 0 ? -1 : 1) * frac, next: dp } + } + } + } + + if (dozen) { return withHalfDozenTail(tokens, pos, value, tables) } - // "two and a half" const tail = parseAndFractionTail(tokens, pos, tables) if (tail) { value += tail.add @@ -211,20 +323,12 @@ function finishScaled( tables: NumberWordTables, ): WordNumberResult { const scaleWord = word(tokens[scalePos])! - let value = multiplier * tables.scales[scaleWord]! - let pos = scalePos + 1 - const join = word(tokens[pos]) - if (join && tables.andWords.has(join)) { - const after = word(tokens[pos + 1]) - if (after && (tables.ones[after] !== undefined || tables.tens[after] !== undefined)) { - const rest = parseCore(tokens, pos + 1, tables) - if (rest && !rest.needsUnit) { - value += rest.value - pos = rest.next - } - } - } - return { value, next: pos } + const scaleValue = multiplier * tables.scales[scaleWord]! + const pos = scalePos + 1 + // Continue into cardinalLoop to consume further composed/scale chains. + // E.g. "un millon quinientos mil" seeds total with 1,000,000, then + // cardinalLoop picks up "quinientos mil" (500*1000). + return cardinalLoop(tokens, pos, scaleValue, 0, true, tables)! } /** diff --git a/packages/lingo/src/parse/normalize.ts b/packages/lingo/src/parse/normalize.ts index 30b9573..0516073 100644 --- a/packages/lingo/src/parse/normalize.ts +++ b/packages/lingo/src/parse/normalize.ts @@ -99,6 +99,9 @@ function foldChar(cp: number): string | null { if (cp === 0x22_12) { return '-' // minus sign } + if (cp === 0x30_1c || cp === 0xff_5e) { + return '–' + } // Apostrophe-alikes → ' (incl. prime U+2032, which NFKC would leave alone, // and acute/backtick, which people type for feet). if ( diff --git a/packages/lingo/src/parse/quantity.ts b/packages/lingo/src/parse/quantity.ts index 950a0c7..c0863c1 100644 --- a/packages/lingo/src/parse/quantity.ts +++ b/packages/lingo/src/parse/quantity.ts @@ -112,6 +112,18 @@ export function parseQualifiers(p: ParserState, i: number): Quals { pos = softener continue } + // Multi-word approximate phrases ("más o menos", "à peu près") — + // matched longest-first before single-word approximateWords. + const approxPhrase = eatAnyPhrase(p, pos, p.profile.grammar.approximatePhrases) + if (approxPhrase >= 0) { + approximate = true + pos = approxPhrase + const skipWord = wordAt(p, pos) + if (skipWord && p.profile.grammar.qualifierSkipAfterApprox.has(skipWord)) { + pos++ + } + continue + } if (p.profile.grammar.approximateWords.has(w)) { approximate = true pos++ diff --git a/packages/lingo/src/parse/range.ts b/packages/lingo/src/parse/range.ts index dda0d47..103575b 100644 --- a/packages/lingo/src/parse/range.ts +++ b/packages/lingo/src/parse/range.ts @@ -2,6 +2,7 @@ import { toBase } from '../core/convert' import { hasError, makeIssue } from '../core/errors' import { Quantity, QuantityRange } from '../core/quantity' import type { Kind } from '../core/types' +import { consumeCjkPostUnitHalf, prepareCjkValueTokens } from '../number/cjk' import { parseValue } from '../number/value' import { type Alternative, @@ -25,9 +26,9 @@ import { isCurrencyPrefixHit, matchCurrencyMinor, minorCurrencyScale, - parseQty, - parseQtyWithQuals, + parseQty as parseQtyBase, parseQualifiers, + type QtyFlags, type QtyNode, type QualifiedQty, resolveImplied, @@ -118,6 +119,11 @@ export function parseRangeOrQty(p: ParserState, i: number, atStart: boolean): Pa } } + const adjacent = tryAdjacentCjkRange(p, a, i) + if (adjacent) { + return adjacent + } + // Spread from fuzzy amounts ("a few minutes" → 2–4 min). if (a.spread && a.kind && a.headUnit) { const unit = p.reg.unit(a.kind, a.headUnit)! @@ -133,6 +139,71 @@ export function parseRangeOrQty(p: ParserState, i: number, atStart: boolean): Pa return singleResult(p, a, quals.approximate) } +function parseQty( + p: ParserState, + i: number, + atStart: boolean, + expectKind?: Kind, + flags?: QtyFlags, +): QtyNode | null { + prepareCjkValueTokens(p.tokens, i, p.profile.numberWords) + const q = parseQtyBase(p, i, atStart, expectKind, flags) + return q ? withCjkPostUnitHalf(p, q) : null +} + +function parseQtyWithQuals( + p: ParserState, + i: number, + atStart: boolean, + flags?: QtyFlags, +): QualifiedQty | null { + const quals = parseQualifiers(p, i) + const q = parseQty(p, quals.next, atStart, undefined, flags) + if (!q) { + return null + } + const qualified = q as QualifiedQty + qualified.quals = quals + if (quals.approximate) { + qualified.approximate = true + } + return qualified +} + +function withCjkPostUnitHalf(p: ParserState, q: QtyNode): QtyNode { + const half = + q.kind && q.headUnit && consumeCjkPostUnitHalf(p.tokens, q.nextToken, p.profile.numberWords) + if (!half) { + return q + } + const part = q.parts[q.parts.length - 1] + const unit = p.reg.unit(q.kind!, part?.unit ?? q.headUnit!) + if (!unit) { + return q + } + q.base += 0.5 * unit.factor + if (part?.unit === unit.id) { + part.value += 0.5 + } else { + q.parts.push({ unit: unit.id, value: 0.5 }) + } + q.normEnd = half.end + q.nextToken = half.next + return q +} + +function tryAdjacentCjkRange(p: ParserState, a: QtyNode, exprStart: number): Parsed | null { + const next = p.tokens[a.nextToken] + if (!next || next.spaceBefore || a.value.value < 1 || a.value.value >= 9) { + return null + } + const b = parseQty(p, a.nextToken, false) + if (!b || b.value.value !== a.value.value + 1) { + return null + } + return buildRange(p, a as QualifiedQty, b as QualifiedQty, exprStart) +} + /** "-", "–", "—", "to", "..", "..." between two values. */ function rangeSeparator(p: ParserState, i: number): number { const t = p.tokens[i] diff --git a/packages/lingo/tests/corpus/contract-v1.json b/packages/lingo/tests/corpus/contract-v1.json index 2d1d533..035e7e0 100644 --- a/packages/lingo/tests/corpus/contract-v1.json +++ b/packages/lingo/tests/corpus/contract-v1.json @@ -2513,6 +2513,32 @@ }, "confidence": 1 }, + "two point five six kg": { + "input": "two point five six kg", + "type": "quantity", + "kind": "mass", + "base": 2.56, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 21 + }, + "confidence": 1 + }, + "three point one four kg": { + "input": "three point one four kg", + "type": "quantity", + "kind": "mass", + "base": 3.14, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 23 + }, + "confidence": 1 + }, "": { "input": "", "type": "fail", diff --git a/packages/lingo/tests/corpus/locale-contract.test.mjs b/packages/lingo/tests/corpus/locale-contract.test.mjs new file mode 100644 index 0000000..5d99c44 --- /dev/null +++ b/packages/lingo/tests/corpus/locale-contract.test.mjs @@ -0,0 +1,32 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { createLingo } from '../../src/index.ts' +import { parseDate, parseDateRange } from '../../src/date/index.ts' +import { es } from '../../src/locales/es.ts' +import { fr } from '../../src/locales/fr.ts' +import { pt } from '../../src/locales/pt.ts' +import { zh } from '../../src/locales/zh.ts' +import { ja } from '../../src/locales/ja.ts' + +const localePacks = [es, fr, pt, zh, ja] +const localeInstance = createLingo({ locales: localePacks }) +const lingo = localeInstance.parse +const corpusDir = new URL('.', import.meta.url) +const sourceFiles = readdirSync(corpusDir) + .filter((f) => /^locale-[a-z]+-source\.mjs$/.test(f)) + .sort() + +describe('locale corpus contracts', () => { + for (const sourceFile of sourceFiles) { + const id = sourceFile.replace('locale-', '').replace('-source.mjs', '') + const contractFile = `locale-${id}-contract-v1.json` + + it(`[${id}] matches the checked-in compatibility snapshot`, async () => { + const mod = await import(`./${sourceFile}`) + const contractPath = new URL(contractFile, corpusDir) + const expected = JSON.parse(readFileSync(contractPath, 'utf8')) + const current = mod.buildContract({ lingo, localePacks, parseDate, parseDateRange }) + expect(current).toEqual(expected) + }) + } +}) diff --git a/packages/lingo/tests/corpus/locale-es-contract-v1.json b/packages/lingo/tests/corpus/locale-es-contract-v1.json new file mode 100644 index 0000000..ea5b1e2 --- /dev/null +++ b/packages/lingo/tests/corpus/locale-es-contract-v1.json @@ -0,0 +1,946 @@ +{ + "version": 1, + "fixedNow": [ + 2026, + 6, + 3, + 14, + 30, + 0 + ], + "breadth": { + "dos kg [basic-word-pin]": { + "input": "dos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 2, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "entre 5 y 10 kg [range-pin]": { + "input": "entre 5 y 10 kg", + "opts": { + "locale": "es" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 6, + "end": 15 + }, + "confidence": 1 + }, + "al menos 2 kg [bound-pin]": { + "input": "al menos 2 kg", + "opts": { + "locale": "es" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 2, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "aproximadamente 5 kg [approx-word-pin]": { + "input": "aproximadamente 5 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 16, + "end": 20 + }, + "confidence": 0.9 + }, + "veintiuno kg": { + "input": "veintiuno kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 21, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "veintiun kg": { + "input": "veintiun kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 21, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "veintidos kg": { + "input": "veintidos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 22, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "veinticinco kg": { + "input": "veinticinco kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 25, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "veintinueve kg": { + "input": "veintinueve kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 29, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "doscientos kg": { + "input": "doscientos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 200, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "doscientas kg": { + "input": "doscientas kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 200, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "trescientos kg": { + "input": "trescientos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 300, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "quinientos kg": { + "input": "quinientos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "quinientas kg": { + "input": "quinientas kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "novecientos kg": { + "input": "novecientos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 900, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "novecientas kg": { + "input": "novecientas kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 900, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "cien kg": { + "input": "cien kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 100, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 7 + }, + "confidence": 1 + }, + "mil kg": { + "input": "mil kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 1000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "ciento cincuenta kg": { + "input": "ciento cincuenta kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 150, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "mil quinientos kg": { + "input": "mil quinientos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 1500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 17 + }, + "confidence": 1 + }, + "dos coma cinco kg": { + "input": "dos coma cinco kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 2.5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 17 + }, + "confidence": 1 + }, + "dos coma cinco seis kg": { + "input": "dos coma cinco seis kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 2.56, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 22 + }, + "confidence": 1 + }, + "quinientos coma cero cinco kg": { + "input": "quinientos coma cero cinco kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 500.05, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 29 + }, + "confidence": 1 + }, + "treinta y cinco kg": { + "input": "treinta y cinco kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 35, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 18 + }, + "confidence": 1 + }, + "cuarenta y dos kg": { + "input": "cuarenta y dos kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 42, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 17 + }, + "confidence": 1 + }, + "noventa y nueve kg": { + "input": "noventa y nueve kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 99, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 18 + }, + "confidence": 1 + }, + "mas o menos 5 kg": { + "input": "mas o menos 5 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 12, + "end": 16 + }, + "confidence": 0.9 + }, + "por ahi de 5 kg": { + "input": "por ahi de 5 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 11, + "end": 15 + }, + "confidence": 0.9 + }, + "alrededor de 5 kg": { + "input": "alrededor de 5 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 13, + "end": 17 + }, + "confidence": 0.9 + }, + "cerca de 5 kg": { + "input": "cerca de 5 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 9, + "end": 13 + }, + "confidence": 0.9 + }, + "cosa de 5 kg": { + "input": "cosa de 5 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 8, + "end": 12 + }, + "confidence": 0.9 + }, + "unos 5 kg": { + "input": "unos 5 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 5, + "end": 9 + }, + "confidence": 0.9 + }, + "unas 3 kg": { + "input": "unas 3 kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 3, + "unit": "kg", + "issues": [], + "span": { + "start": 5, + "end": 9 + }, + "confidence": 0.9 + }, + "5 kg y pico": { + "input": "5 kg y pico", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 kg y tantos": { + "input": "5 kg y tantos", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 kg y algo": { + "input": "5 kg y algo", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 o 6 kg": { + "input": "5 o 6 kg", + "opts": { + "locale": "es" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 6 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 8 + }, + "confidence": 1 + }, + "de 5 a 10 kg": { + "input": "de 5 a 10 kg", + "opts": { + "locale": "es" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 3, + "end": 12 + }, + "confidence": 1 + }, + "un par de kg": { + "input": "un par de kg", + "opts": { + "locale": "es" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 2, + "max": 3 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 0.9 + } + }, + "date": { + "hace tres dias [relative-past-pin]": { + "input": "hace tres dias", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 30, + "hour": 14, + "minute": 30, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "manana [tomorrow-pin]": { + "input": "manana", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "ayer [yesterday-pin]": { + "input": "ayer", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 2, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "pasado manana": { + "input": "pasado manana", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "anteayer": { + "input": "anteayer", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 8 + }, + "confidence": 1 + }, + "antier": { + "input": "antier", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "las dos y media": { + "input": "las dos y media", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 2, + "minute": 30, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "las tres menos cuarto": { + "input": "las tres menos cuarto", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 2, + "minute": 45, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 21 + }, + "confidence": 1 + }, + "a principios de mes": { + "input": "a principios de mes", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "a mediados de julio": { + "input": "a mediados de julio", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 15, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "a finales de mes": { + "input": "a finales de mes", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "a finales de julio": { + "input": "a finales de julio", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 18 + }, + "confidence": 1 + } + }, + "dateRange": {} +} diff --git a/packages/lingo/tests/corpus/locale-es-source.mjs b/packages/lingo/tests/corpus/locale-es-source.mjs new file mode 100644 index 0000000..624bf01 --- /dev/null +++ b/packages/lingo/tests/corpus/locale-es-source.mjs @@ -0,0 +1,233 @@ +export const FIXED_NOW = [2026, 6, 3, 14, 30, 0] + +export const breadthRows = [ + // Pre-existing pack behavior + ['dos kg', { locale: 'es' }, 'basic-word-pin'], + ['entre 5 y 10 kg', { locale: 'es' }, 'range-pin'], + ['al menos 2 kg', { locale: 'es' }, 'bound-pin'], + ['aproximadamente 5 kg', { locale: 'es' }, 'approx-word-pin'], + // Composed: fused veinti- forms (21-29) + ['veintiuno kg', { locale: 'es' }], + ['veintiun kg', { locale: 'es' }], + ['veintidos kg', { locale: 'es' }], + ['veinticinco kg', { locale: 'es' }], + ['veintinueve kg', { locale: 'es' }], + // Composed: compound hundreds (200-900) + ['doscientos kg', { locale: 'es' }], + ['doscientas kg', { locale: 'es' }], + ['trescientos kg', { locale: 'es' }], + ['quinientos kg', { locale: 'es' }], + ['quinientas kg', { locale: 'es' }], + ['novecientos kg', { locale: 'es' }], + ['novecientas kg', { locale: 'es' }], + // bareScales + ['cien kg', { locale: 'es' }], + ['mil kg', { locale: 'es' }], + ['ciento cincuenta kg', { locale: 'es' }], + ['mil quinientos kg', { locale: 'es' }], + // decimalWords + ['dos coma cinco kg', { locale: 'es' }], + ['dos coma cinco seis kg', { locale: 'es' }], + ['quinientos coma cero cinco kg', { locale: 'es' }], + // tens + y + ones composition + ['treinta y cinco kg', { locale: 'es' }], + ['cuarenta y dos kg', { locale: 'es' }], + ['noventa y nueve kg', { locale: 'es' }], + // approximatePhrases + ['mas o menos 5 kg', { locale: 'es' }], + ['por ahi de 5 kg', { locale: 'es' }], + ['alrededor de 5 kg', { locale: 'es' }], + ['cerca de 5 kg', { locale: 'es' }], + ['cosa de 5 kg', { locale: 'es' }], + // approximateWords additions + ['unos 5 kg', { locale: 'es' }], + ['unas 3 kg', { locale: 'es' }], + // trailingApproxPhrases + ['5 kg y pico', { locale: 'es' }], + ['5 kg y tantos', { locale: 'es' }], + ['5 kg y algo', { locale: 'es' }], + // rangeAlternativeWords + ['5 o 6 kg', { locale: 'es' }], + // rangeFromWords 'de' + ['de 5 a 10 kg', { locale: 'es' }], + // fuzzyAmounts + ['un par de kg', { locale: 'es' }], +] + +export const dateRows = [ + // Pre-existing date behavior + ['hace tres dias', { locale: 'es' }, 'relative-past-pin'], + ['manana', { locale: 'es' }, 'tomorrow-pin'], + ['ayer', { locale: 'es' }, 'yesterday-pin'], + // dayOffsets: +2, -2 + ['pasado manana', { locale: 'es' }], + ['anteayer', { locale: 'es' }], + ['antier', { locale: 'es' }], + // localized spoken clock + ['las dos y media', { locale: 'es' }], + ['las tres menos cuarto', { locale: 'es' }], + // localized period edges + ['a principios de mes', { locale: 'es' }], + ['a mediados de julio', { locale: 'es' }], + ['a finales de mes', { locale: 'es' }], + ['a finales de julio', { locale: 'es' }], +] + +export const dateRangeRows = [] + +export function buildContract({ lingo, localePacks, parseDate, parseDateRange }) { + return { + version: 1, + fixedNow: FIXED_NOW, + breadth: entriesFromRows(breadthRows, (input, opts) => lingo(input, opts)), + date: entriesFromRows( + dateRows, + (input, opts) => parseDate(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + dateRange: entriesFromRows( + dateRangeRows, + (input, opts) => parseDateRange(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + } +} + +function entriesFromRows(rows, parse, options = {}) { + const entries = {} + for (const row of rows) { + const [input, opts, label] = row + const key = uniqueKey(input, opts, label) + const parseOpts = options.defaultNow ? { now: FIXED_NOW, ...(opts ?? {}) } : opts + entries[key] = { + input, + ...(opts ? { opts } : {}), + ...summarize(parse(input, parseOpts)), + } + } + return entries +} + +function uniqueKey(input, opts, label) { + if (label) return `${input} [${label}]` + if (!opts?.now) return input + return `${input} [now:${opts.now.join(',')}]` +} + +function reviveDateOptions(opts, localePacks) { + if (!opts) return { ...(localePacks ? { localePacks } : {}), now: dateFromParts(FIXED_NOW) } + const out = { ...(localePacks ? { localePacks } : {}), ...opts } + const now = opts.now ?? FIXED_NOW + out.now = dateFromParts(now) + return out +} + +function summarize(result) { + const base = { + type: result.ok ? result.type : 'fail', + kind: null, + base: null, + unit: null, + issues: result.issues.map((issue) => issue.code), + span: result.ok ? { ...result.span } : null, + confidence: result.ok ? result.confidence : null, + } + if (!result.ok) return base + if (result.type === 'quantity') return { ...base, ...quantityShape(result.quantity) } + if (result.type === 'range') return { ...base, ...rangeShape(result.range) } + if (result.type === 'conversion') { + return { + ...base, + kind: result.converted.kind, + base: { + source: valueShape(result.source), + converted: valueShape(result.converted), + }, + unit: result.targetUnit, + } + } + if (result.type === 'number') { + return { ...base, base: number(result.value), unit: null } + } + if (result.type === 'date') { + return { + ...base, + kind: 'date', + base: dateParts(result.date), + unit: result.grain, + } + } + if (result.type === 'date-range') { + return { + ...base, + kind: 'date-range', + base: { + start: result.start ? dateParts(result.start.date) : null, + end: result.end ? dateParts(result.end.date) : null, + }, + unit: { + start: result.start ? result.start.grain : null, + end: result.end ? result.end.grain : null, + }, + } + } + return base +} + +function valueShape(value) { + return 'base' in value ? quantityShape(value) : rangeShape(value) +} + +function quantityShape(quantity) { + return { + kind: quantity.kind, + base: number(quantity.base), + unit: quantity.unit, + } +} + +function rangeShape(range) { + return { + kind: range.kind, + base: { + min: nullableNumber(range.minBase), + max: nullableNumber(range.maxBase), + ...(range.plusMinus + ? { + plusMinus: { + center: number(range.plusMinus.centerBase), + delta: number(range.plusMinus.deltaBase), + }, + } + : {}), + }, + unit: { + min: range.minUnit, + max: range.maxUnit, + ...(range.plusMinus ? { plusMinus: range.plusMinus.unit } : {}), + }, + } +} + +function dateFromParts(parts) { + return new Date(...parts) +} + +function dateParts(date) { + return { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + hour: date.getHours(), + minute: date.getMinutes(), + second: date.getSeconds(), + } +} + +function nullableNumber(value) { + return value === null ? null : number(value) +} + +function number(value) { + return Number(Number(value).toPrecision(15)) +} diff --git a/packages/lingo/tests/corpus/locale-fr-contract-v1.json b/packages/lingo/tests/corpus/locale-fr-contract-v1.json new file mode 100644 index 0000000..c90f41f --- /dev/null +++ b/packages/lingo/tests/corpus/locale-fr-contract-v1.json @@ -0,0 +1,945 @@ +{ + "version": 1, + "fixedNow": [ + 2026, + 6, + 3, + 14, + 30, + 0 + ], + "breadth": { + "deux kg [basic-word-pin]": { + "input": "deux kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 2, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 7 + }, + "confidence": 1 + }, + "entre 5 et 10 kg [range-pin]": { + "input": "entre 5 et 10 kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 6, + "end": 16 + }, + "confidence": 1 + }, + "au moins 2 kg [bound-pin]": { + "input": "au moins 2 kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 2, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "environ 5 kg [approx-word-pin]": { + "input": "environ 5 kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 8, + "end": 12 + }, + "confidence": 0.9 + }, + "soixante-dix kg": { + "input": "soixante-dix kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 70, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "soixante-quinze kg": { + "input": "soixante-quinze kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 75, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 18 + }, + "confidence": 1 + }, + "soixante-dix-neuf kg": { + "input": "soixante-dix-neuf kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 79, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 20 + }, + "confidence": 1 + }, + "quatre-vingts kg": { + "input": "quatre-vingts kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 80, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "quatre-vingt-cinq kg": { + "input": "quatre-vingt-cinq kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 85, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 20 + }, + "confidence": 1 + }, + "quatre-vingt-neuf kg": { + "input": "quatre-vingt-neuf kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 89, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 20 + }, + "confidence": 1 + }, + "quatre-vingt-dix kg": { + "input": "quatre-vingt-dix kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 90, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "quatre-vingt-quinze kg": { + "input": "quatre-vingt-quinze kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 95, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 22 + }, + "confidence": 1 + }, + "quatre-vingt-dix-neuf kg": { + "input": "quatre-vingt-dix-neuf kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 99, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 24 + }, + "confidence": 1 + }, + "septante-cinq kg": { + "input": "septante-cinq kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 75, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "nonante-deux kg": { + "input": "nonante-deux kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 92, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "huitante kg": { + "input": "huitante kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 80, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "cent kg": { + "input": "cent kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 100, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 7 + }, + "confidence": 1 + }, + "mille kg": { + "input": "mille kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 1000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 8 + }, + "confidence": 1 + }, + "vingt et un kg": { + "input": "vingt et un kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 21, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "trente et un kg": { + "input": "trente et un kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 31, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "deux virgule cinq kg": { + "input": "deux virgule cinq kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 2.5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 20 + }, + "confidence": 1 + }, + "trois virgule quatorze kg": { + "input": "trois virgule quatorze kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 3.14, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 25 + }, + "confidence": 1 + }, + "deux virgule cinq six kg": { + "input": "deux virgule cinq six kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 2.56, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 24 + }, + "confidence": 1 + }, + "a peu pres 5 kg": { + "input": "a peu pres 5 kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 11, + "end": 15 + }, + "confidence": 0.9 + }, + "plus ou moins 5 kg": { + "input": "plus ou moins 5 kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 14, + "end": 18 + }, + "confidence": 0.9 + }, + "grosso modo 5 kg": { + "input": "grosso modo 5 kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 12, + "end": 16 + }, + "confidence": 0.9 + }, + "5 kg environ": { + "input": "5 kg environ", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 kg et quelques": { + "input": "5 kg et quelques", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 kg a peu pres": { + "input": "5 kg a peu pres", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 ou 6 kg": { + "input": "5 ou 6 kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 6 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 9 + }, + "confidence": 1 + }, + "une dizaine de kg": { + "input": "une dizaine de kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 8, + "max": 12 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 17 + }, + "confidence": 0.9 + }, + "une vingtaine de kg": { + "input": "une vingtaine de kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 18, + "max": 25 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 0.9 + }, + "une trentaine de kg": { + "input": "une trentaine de kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 25, + "max": 35 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 0.9 + }, + "une centaine de kg": { + "input": "une centaine de kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 80, + "max": 120 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 18 + }, + "confidence": 0.9 + }, + "un millier de kg": { + "input": "un millier de kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 800, + "max": 1200 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 0.9 + }, + "quelques kg": { + "input": "quelques kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 2, + "max": 4 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 0.9 + } + }, + "date": { + "il y a trois jours [relative-past-pin]": { + "input": "il y a trois jours", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 30, + "hour": 14, + "minute": 30, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 18 + }, + "confidence": 1 + }, + "demain [tomorrow-pin]": { + "input": "demain", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "hier [yesterday-pin]": { + "input": "hier", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 2, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "apres-demain": { + "input": "apres-demain", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "avant-hier": { + "input": "avant-hier", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 10 + }, + "confidence": 1 + }, + "deux heures et quart": { + "input": "deux heures et quart", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 2, + "minute": 15, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 20 + }, + "confidence": 1 + }, + "trois heures moins le quart": { + "input": "trois heures moins le quart", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 2, + "minute": 45, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 27 + }, + "confidence": 1 + }, + "midi et demi": { + "input": "midi et demi", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 12, + "minute": 30, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "debut juillet": { + "input": "debut juillet", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "mi-juillet": { + "input": "mi-juillet", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 15, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 10 + }, + "confidence": 1 + }, + "fin juillet": { + "input": "fin juillet", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "lundi en huit": { + "input": "lundi en huit", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 13, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "mardi en quinze": { + "input": "mardi en quinze", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 21, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + } + }, + "dateRange": {} +} diff --git a/packages/lingo/tests/corpus/locale-fr-source.mjs b/packages/lingo/tests/corpus/locale-fr-source.mjs new file mode 100644 index 0000000..2dba26f --- /dev/null +++ b/packages/lingo/tests/corpus/locale-fr-source.mjs @@ -0,0 +1,233 @@ +export const FIXED_NOW = [2026, 6, 3, 14, 30, 0] + +export const breadthRows = [ + // Pre-existing pack behavior + ['deux kg', { locale: 'fr' }, 'basic-word-pin'], + ['entre 5 et 10 kg', { locale: 'fr' }, 'range-pin'], + ['au moins 2 kg', { locale: 'fr' }, 'bound-pin'], + ['environ 5 kg', { locale: 'fr' }, 'approx-word-pin'], + // Composed: vigesimal (70-79) + ['soixante-dix kg', { locale: 'fr' }], + ['soixante-quinze kg', { locale: 'fr' }], + ['soixante-dix-neuf kg', { locale: 'fr' }], + // Composed: vigesimal (80-89) + ['quatre-vingts kg', { locale: 'fr' }], + ['quatre-vingt-cinq kg', { locale: 'fr' }], + ['quatre-vingt-neuf kg', { locale: 'fr' }], + // Composed: vigesimal (90-99) + ['quatre-vingt-dix kg', { locale: 'fr' }], + ['quatre-vingt-quinze kg', { locale: 'fr' }], + ['quatre-vingt-dix-neuf kg', { locale: 'fr' }], + // Regional (Belgian/Swiss) — already worked + ['septante-cinq kg', { locale: 'fr' }], + ['nonante-deux kg', { locale: 'fr' }], + ['huitante kg', { locale: 'fr' }], + // bareScales + ['cent kg', { locale: 'fr' }], + ['mille kg', { locale: 'fr' }], + // tens + et + ones + ['vingt et un kg', { locale: 'fr' }], + ['trente et un kg', { locale: 'fr' }], + // decimalWords + ['deux virgule cinq kg', { locale: 'fr' }], + ['trois virgule quatorze kg', { locale: 'fr' }], + ['deux virgule cinq six kg', { locale: 'fr' }], + // approximatePhrases + ['a peu pres 5 kg', { locale: 'fr' }], + ['plus ou moins 5 kg', { locale: 'fr' }], + ['grosso modo 5 kg', { locale: 'fr' }], + // trailingApproxWords + ['5 kg environ', { locale: 'fr' }], + // trailingApproxPhrases + ['5 kg et quelques', { locale: 'fr' }], + ['5 kg a peu pres', { locale: 'fr' }], + // rangeAlternativeWords + ['5 ou 6 kg', { locale: 'fr' }], + // fuzzyAmounts + ['une dizaine de kg', { locale: 'fr' }], + ['une vingtaine de kg', { locale: 'fr' }], + ['une trentaine de kg', { locale: 'fr' }], + ['une centaine de kg', { locale: 'fr' }], + ['un millier de kg', { locale: 'fr' }], + ['quelques kg', { locale: 'fr' }], +] + +export const dateRows = [ + // Pre-existing date behavior + ['il y a trois jours', { locale: 'fr' }, 'relative-past-pin'], + ['demain', { locale: 'fr' }, 'tomorrow-pin'], + ['hier', { locale: 'fr' }, 'yesterday-pin'], + // dayOffsets: +2, -2 + ['apres-demain', { locale: 'fr' }], + ['avant-hier', { locale: 'fr' }], + // localized spoken clock + ['deux heures et quart', { locale: 'fr' }], + ['trois heures moins le quart', { locale: 'fr' }], + ['midi et demi', { locale: 'fr' }], + // localized period edges + ['debut juillet', { locale: 'fr' }], + ['mi-juillet', { locale: 'fr' }], + ['fin juillet', { locale: 'fr' }], + // localized weekday offsets + ['lundi en huit', { locale: 'fr' }], + ['mardi en quinze', { locale: 'fr' }], +] + +export const dateRangeRows = [] + +export function buildContract({ lingo, localePacks, parseDate, parseDateRange }) { + return { + version: 1, + fixedNow: FIXED_NOW, + breadth: entriesFromRows(breadthRows, (input, opts) => lingo(input, opts)), + date: entriesFromRows( + dateRows, + (input, opts) => parseDate(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + dateRange: entriesFromRows( + dateRangeRows, + (input, opts) => parseDateRange(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + } +} + +function entriesFromRows(rows, parse, options = {}) { + const entries = {} + for (const row of rows) { + const [input, opts, label] = row + const key = uniqueKey(input, opts, label) + const parseOpts = options.defaultNow ? { now: FIXED_NOW, ...(opts ?? {}) } : opts + entries[key] = { + input, + ...(opts ? { opts } : {}), + ...summarize(parse(input, parseOpts)), + } + } + return entries +} + +function uniqueKey(input, opts, label) { + if (label) return `${input} [${label}]` + if (!opts?.now) return input + return `${input} [now:${opts.now.join(',')}]` +} + +function reviveDateOptions(opts, localePacks) { + if (!opts) return { ...(localePacks ? { localePacks } : {}), now: dateFromParts(FIXED_NOW) } + const out = { ...(localePacks ? { localePacks } : {}), ...opts } + const now = opts.now ?? FIXED_NOW + out.now = dateFromParts(now) + return out +} + +function summarize(result) { + const base = { + type: result.ok ? result.type : 'fail', + kind: null, + base: null, + unit: null, + issues: result.issues.map((issue) => issue.code), + span: result.ok ? { ...result.span } : null, + confidence: result.ok ? result.confidence : null, + } + if (!result.ok) return base + if (result.type === 'quantity') return { ...base, ...quantityShape(result.quantity) } + if (result.type === 'range') return { ...base, ...rangeShape(result.range) } + if (result.type === 'conversion') { + return { + ...base, + kind: result.converted.kind, + base: { + source: valueShape(result.source), + converted: valueShape(result.converted), + }, + unit: result.targetUnit, + } + } + if (result.type === 'number') { + return { ...base, base: number(result.value), unit: null } + } + if (result.type === 'date') { + return { + ...base, + kind: 'date', + base: dateParts(result.date), + unit: result.grain, + } + } + if (result.type === 'date-range') { + return { + ...base, + kind: 'date-range', + base: { + start: result.start ? dateParts(result.start.date) : null, + end: result.end ? dateParts(result.end.date) : null, + }, + unit: { + start: result.start ? result.start.grain : null, + end: result.end ? result.end.grain : null, + }, + } + } + return base +} + +function valueShape(value) { + return 'base' in value ? quantityShape(value) : rangeShape(value) +} + +function quantityShape(quantity) { + return { + kind: quantity.kind, + base: number(quantity.base), + unit: quantity.unit, + } +} + +function rangeShape(range) { + return { + kind: range.kind, + base: { + min: nullableNumber(range.minBase), + max: nullableNumber(range.maxBase), + ...(range.plusMinus + ? { + plusMinus: { + center: number(range.plusMinus.centerBase), + delta: number(range.plusMinus.deltaBase), + }, + } + : {}), + }, + unit: { + min: range.minUnit, + max: range.maxUnit, + ...(range.plusMinus ? { plusMinus: range.plusMinus.unit } : {}), + }, + } +} + +function dateFromParts(parts) { + return new Date(...parts) +} + +function dateParts(date) { + return { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + hour: date.getHours(), + minute: date.getMinutes(), + second: date.getSeconds(), + } +} + +function nullableNumber(value) { + return value === null ? null : number(value) +} + +function number(value) { + return Number(Number(value).toPrecision(15)) +} diff --git a/packages/lingo/tests/corpus/locale-ja-contract-v1.json b/packages/lingo/tests/corpus/locale-ja-contract-v1.json new file mode 100644 index 0000000..c3f040d --- /dev/null +++ b/packages/lingo/tests/corpus/locale-ja-contract-v1.json @@ -0,0 +1,690 @@ +{ + "version": 1, + "fixedNow": [ + 2026, + 6, + 3, + 14, + 30, + 0 + ], + "breadth": { + "5キロ [kg-arabic-pin]": { + "input": "5キロ", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "三キロ": { + "input": "三キロ", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 3, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "三十五キロ": { + "input": "三十五キロ", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 35, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "百五十グラム": { + "input": "百五十グラム", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 0.15, + "unit": "g", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "千五百メートル": { + "input": "千五百メートル", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "length", + "base": 1500, + "unit": "m", + "issues": [], + "span": { + "start": 0, + "end": 7 + }, + "confidence": 1 + }, + "三万五千円": { + "input": "三万五千円", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "currency", + "base": 35000, + "unit": "JPY", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "3万5千キロ": { + "input": "3万5千キロ", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 35000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "二億三千万": { + "input": "二億三千万", + "opts": { + "locale": "ja" + }, + "type": "number", + "kind": null, + "base": 230000000, + "unit": null, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "二億三千万円": { + "input": "二億三千万円", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "currency", + "base": 230000000, + "unit": "JPY", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "二キロ半": { + "input": "二キロ半", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 2.5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "一時間半": { + "input": "一時間半", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "duration", + "base": 5400, + "unit": "h", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "３５キロ": { + "input": "３５キロ", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 35, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "5〜10キロ": { + "input": "5〜10キロ", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "5～10キロ": { + "input": "5～10キロ", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "三〜五日": { + "input": "三〜五日", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "duration", + "base": { + "min": 259200, + "max": 432000 + }, + "unit": { + "min": "d", + "max": "d" + }, + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "約5キロ": { + "input": "約5キロ", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 1, + "end": 4 + }, + "confidence": 0.9 + }, + "5キロ ほど": { + "input": "5キロ ほど", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 0.9 + }, + "三個": { + "input": "三個", + "opts": { + "locale": "ja" + }, + "type": "number", + "kind": null, + "base": 3, + "unit": null, + "issues": [], + "span": { + "start": 0, + "end": 1 + }, + "confidence": 1 + } + }, + "date": { + "3日前 [compact-days-ago-pin]": { + "input": "3日前", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 30, + "hour": 14, + "minute": 30, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "一昨日": { + "input": "一昨日", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "おととい": { + "input": "おととい", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "明後日": { + "input": "明後日", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "あさって": { + "input": "あさって", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "しあさって": { + "input": "しあさって", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "今朝": { + "input": "今朝", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 8, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "今晩": { + "input": "今晩", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 19, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "今夜": { + "input": "今夜", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 21, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "来週": { + "input": "来週", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "week", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "再来週": { + "input": "再来週", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 13, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "week", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "再来月": { + "input": "再来月", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 9, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "month", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "先々週": { + "input": "先々週", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 15, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "week", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "先々月": { + "input": "先々月", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 5, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "month", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "週末": { + "input": "週末", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "月末": { + "input": "月末", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + } + }, + "dateRange": {} +} diff --git a/packages/lingo/tests/corpus/locale-ja-source.mjs b/packages/lingo/tests/corpus/locale-ja-source.mjs new file mode 100644 index 0000000..620fa79 --- /dev/null +++ b/packages/lingo/tests/corpus/locale-ja-source.mjs @@ -0,0 +1,200 @@ +export const FIXED_NOW = [2026, 6, 3, 14, 30, 0] + +export const breadthRows = [ + ['5キロ', { locale: 'ja' }, 'kg-arabic-pin'], + ['三キロ', { locale: 'ja' }], + ['三十五キロ', { locale: 'ja' }], + ['百五十グラム', { locale: 'ja' }], + ['千五百メートル', { locale: 'ja' }], + ['三万五千円', { locale: 'ja' }], + ['3万5千キロ', { locale: 'ja' }], + ['二億三千万', { locale: 'ja' }], + ['二億三千万円', { locale: 'ja' }], + ['二キロ半', { locale: 'ja' }], + ['一時間半', { locale: 'ja' }], + ['３５キロ', { locale: 'ja' }], + ['5〜10キロ', { locale: 'ja' }], + ['5～10キロ', { locale: 'ja' }], + ['三〜五日', { locale: 'ja' }], + ['約5キロ', { locale: 'ja' }], + ['5キロ ほど', { locale: 'ja' }], + ['三個', { locale: 'ja' }], +] + +export const dateRows = [ + ['3日前', { locale: 'ja' }, 'compact-days-ago-pin'], + ['一昨日', { locale: 'ja' }], + ['おととい', { locale: 'ja' }], + ['明後日', { locale: 'ja' }], + ['あさって', { locale: 'ja' }], + ['しあさって', { locale: 'ja' }], + ['今朝', { locale: 'ja' }], + ['今晩', { locale: 'ja' }], + ['今夜', { locale: 'ja' }], + ['来週', { locale: 'ja' }], + ['再来週', { locale: 'ja' }], + ['再来月', { locale: 'ja' }], + ['先々週', { locale: 'ja' }], + ['先々月', { locale: 'ja' }], + ['週末', { locale: 'ja' }], + ['月末', { locale: 'ja' }], +] + +export const dateRangeRows = [] + +export function buildContract({ lingo, localePacks, parseDate, parseDateRange }) { + return { + version: 1, + fixedNow: FIXED_NOW, + breadth: entriesFromRows(breadthRows, (input, opts) => lingo(input, opts)), + date: entriesFromRows( + dateRows, + (input, opts) => parseDate(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + dateRange: entriesFromRows( + dateRangeRows, + (input, opts) => parseDateRange(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + } +} + +function entriesFromRows(rows, parse, options = {}) { + const entries = {} + for (const row of rows) { + const [input, opts, label] = row + const key = uniqueKey(input, opts, label) + const parseOpts = options.defaultNow ? { now: FIXED_NOW, ...(opts ?? {}) } : opts + entries[key] = { + input, + ...(opts ? { opts } : {}), + ...summarize(parse(input, parseOpts)), + } + } + return entries +} + +function uniqueKey(input, opts, label) { + if (label) return `${input} [${label}]` + if (!opts?.now) return input + return `${input} [now:${opts.now.join(',')}]` +} + +function reviveDateOptions(opts, localePacks) { + if (!opts) return { ...(localePacks ? { localePacks } : {}), now: dateFromParts(FIXED_NOW) } + const out = { ...(localePacks ? { localePacks } : {}), ...opts } + const now = opts.now ?? FIXED_NOW + out.now = dateFromParts(now) + return out +} + +function summarize(result) { + const base = { + type: result.ok ? result.type : 'fail', + kind: null, + base: null, + unit: null, + issues: result.issues.map((issue) => issue.code), + span: result.ok ? { ...result.span } : null, + confidence: result.ok ? result.confidence : null, + } + if (!result.ok) return base + if (result.type === 'quantity') return { ...base, ...quantityShape(result.quantity) } + if (result.type === 'range') return { ...base, ...rangeShape(result.range) } + if (result.type === 'conversion') { + return { + ...base, + kind: result.converted.kind, + base: { + source: valueShape(result.source), + converted: valueShape(result.converted), + }, + unit: result.targetUnit, + } + } + if (result.type === 'number') { + return { ...base, base: number(result.value), unit: null } + } + if (result.type === 'date') { + return { + ...base, + kind: 'date', + base: dateParts(result.date), + unit: result.grain, + } + } + if (result.type === 'date-range') { + return { + ...base, + kind: 'date-range', + base: { + start: result.start ? dateParts(result.start.date) : null, + end: result.end ? dateParts(result.end.date) : null, + }, + unit: { + start: result.start ? result.start.grain : null, + end: result.end ? result.end.grain : null, + }, + } + } + return base +} + +function valueShape(value) { + return 'base' in value ? quantityShape(value) : rangeShape(value) +} + +function quantityShape(quantity) { + return { + kind: quantity.kind, + base: number(quantity.base), + unit: quantity.unit, + } +} + +function rangeShape(range) { + return { + kind: range.kind, + base: { + min: nullableNumber(range.minBase), + max: nullableNumber(range.maxBase), + ...(range.plusMinus + ? { + plusMinus: { + center: number(range.plusMinus.centerBase), + delta: number(range.plusMinus.deltaBase), + }, + } + : {}), + }, + unit: { + min: range.minUnit, + max: range.maxUnit, + ...(range.plusMinus ? { plusMinus: range.plusMinus.unit } : {}), + }, + } +} + +function dateFromParts(parts) { + return new Date(...parts) +} + +function dateParts(date) { + return { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + hour: date.getHours(), + minute: date.getMinutes(), + second: date.getSeconds(), + } +} + +function nullableNumber(value) { + return value === null ? null : number(value) +} + +function number(value) { + return Number(Number(value).toPrecision(15)) +} diff --git a/packages/lingo/tests/corpus/locale-pt-contract-v1.json b/packages/lingo/tests/corpus/locale-pt-contract-v1.json new file mode 100644 index 0000000..7c2f8df --- /dev/null +++ b/packages/lingo/tests/corpus/locale-pt-contract-v1.json @@ -0,0 +1,1102 @@ +{ + "version": 1, + "fixedNow": [ + 2026, + 6, + 3, + 14, + 30, + 0 + ], + "breadth": { + "dois kg [basic-word-pin]": { + "input": "dois kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 2, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 7 + }, + "confidence": 1 + }, + "entre 5 e 10 kg [range-pin]": { + "input": "entre 5 e 10 kg", + "opts": { + "locale": "pt" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 6, + "end": 15 + }, + "confidence": 1 + }, + "pelo menos 2 kg [bound-pin]": { + "input": "pelo menos 2 kg", + "opts": { + "locale": "pt" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 2, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "aproximadamente 5 kg [approx-word-pin]": { + "input": "aproximadamente 5 kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 16, + "end": 20 + }, + "confidence": 0.9 + }, + "duzentos kg": { + "input": "duzentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 200, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "duzentas kg": { + "input": "duzentas kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 200, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "trezentos kg": { + "input": "trezentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 300, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "quatrocentos kg": { + "input": "quatrocentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 400, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "quatrocentas kg": { + "input": "quatrocentas kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 400, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "quinhentos kg": { + "input": "quinhentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "quinhentas kg": { + "input": "quinhentas kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "seiscentos kg": { + "input": "seiscentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 600, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "seiscentas kg": { + "input": "seiscentas kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 600, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "setecentos kg": { + "input": "setecentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 700, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "setecentas kg": { + "input": "setecentas kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 700, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "oitocentos kg": { + "input": "oitocentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 800, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "oitocentas kg": { + "input": "oitocentas kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 800, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "novecentos kg": { + "input": "novecentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 900, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "novecentas kg": { + "input": "novecentas kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 900, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "cem kg": { + "input": "cem kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 100, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "mil kg": { + "input": "mil kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 1000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "vinte e cinco kg": { + "input": "vinte e cinco kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 25, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "trinta e sete kg": { + "input": "trinta e sete kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 37, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "noventa e nove kg": { + "input": "noventa e nove kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 99, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 17 + }, + "confidence": 1 + }, + "dois virgula cinco kg": { + "input": "dois virgula cinco kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 2.5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 21 + }, + "confidence": 1 + }, + "dois virgula cinco seis kg": { + "input": "dois virgula cinco seis kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 2.56, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 26 + }, + "confidence": 1 + }, + "mais ou menos 5 kg": { + "input": "mais ou menos 5 kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 14, + "end": 18 + }, + "confidence": 0.9 + }, + "por volta de 5 kg": { + "input": "por volta de 5 kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 13, + "end": 17 + }, + "confidence": 0.9 + }, + "cerca de 5 kg": { + "input": "cerca de 5 kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 9, + "end": 13 + }, + "confidence": 0.9 + }, + "la pelas 5 kg": { + "input": "la pelas 5 kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 9, + "end": 13 + }, + "confidence": 0.9 + }, + "uns 5 kg": { + "input": "uns 5 kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 4, + "end": 8 + }, + "confidence": 0.9 + }, + "umas 3 kg": { + "input": "umas 3 kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 3, + "unit": "kg", + "issues": [], + "span": { + "start": 5, + "end": 9 + }, + "confidence": 0.9 + }, + "5 kg e pouco": { + "input": "5 kg e pouco", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 kg e poucos": { + "input": "5 kg e poucos", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 kg e tanto": { + "input": "5 kg e tanto", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 0.9 + }, + "5 ou 6 kg": { + "input": "5 ou 6 kg", + "opts": { + "locale": "pt" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 6 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 9 + }, + "confidence": 1 + }, + "de 5 a 10 kg": { + "input": "de 5 a 10 kg", + "opts": { + "locale": "pt" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 3, + "end": 12 + }, + "confidence": 1 + }, + "um par de kg": { + "input": "um par de kg", + "opts": { + "locale": "pt" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 2, + "max": 3 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 0.9 + }, + "dezessete kg": { + "input": "dezessete kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 17, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "dezassete kg": { + "input": "dezassete kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 17, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "dezanove kg": { + "input": "dezanove kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 19, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "dezesseis kg": { + "input": "dezesseis kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 16, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "dezasseis kg": { + "input": "dezasseis kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 16, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + } + }, + "date": { + "ha tres dias [relative-past-pin]": { + "input": "ha tres dias", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 30, + "hour": 14, + "minute": 30, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "amanha [tomorrow-pin]": { + "input": "amanha", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "ontem [yesterday-pin]": { + "input": "ontem", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 2, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "de madrugada": { + "input": "de madrugada", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 4, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "anteontem": { + "input": "anteontem", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 9 + }, + "confidence": 1 + }, + "depois de amanha": { + "input": "depois de amanha", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "duas e meia": { + "input": "duas e meia", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 2, + "minute": 30, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "tres e quinze": { + "input": "tres e quinze", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 15, + "minute": 3, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "quinze para as tres": { + "input": "quinze para as tres", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 2, + "minute": 45, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "dez para as oito": { + "input": "dez para as oito", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 7, + "minute": 50, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "inicio de julho": { + "input": "inicio de julho", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "comeco de julho": { + "input": "comeco de julho", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "no comeco do mes": { + "input": "no comeco do mes", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "meio de julho": { + "input": "meio de julho", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 15, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "fim de julho": { + "input": "fim de julho", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 12 + }, + "confidence": 1 + }, + "final de julho": { + "input": "final de julho", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + } + }, + "dateRange": {} +} diff --git a/packages/lingo/tests/corpus/locale-pt-source.mjs b/packages/lingo/tests/corpus/locale-pt-source.mjs new file mode 100644 index 0000000..7b9d62f --- /dev/null +++ b/packages/lingo/tests/corpus/locale-pt-source.mjs @@ -0,0 +1,241 @@ +export const FIXED_NOW = [2026, 6, 3, 14, 30, 0] + +export const breadthRows = [ + // Pre-existing pack behavior + ['dois kg', { locale: 'pt' }, 'basic-word-pin'], + ['entre 5 e 10 kg', { locale: 'pt' }, 'range-pin'], + ['pelo menos 2 kg', { locale: 'pt' }, 'bound-pin'], + ['aproximadamente 5 kg', { locale: 'pt' }, 'approx-word-pin'], + // Composed: compound hundreds (200-900) + ['duzentos kg', { locale: 'pt' }], + ['duzentas kg', { locale: 'pt' }], + ['trezentos kg', { locale: 'pt' }], + ['quatrocentos kg', { locale: 'pt' }], + ['quatrocentas kg', { locale: 'pt' }], + ['quinhentos kg', { locale: 'pt' }], + ['quinhentas kg', { locale: 'pt' }], + ['seiscentos kg', { locale: 'pt' }], + ['seiscentas kg', { locale: 'pt' }], + ['setecentos kg', { locale: 'pt' }], + ['setecentas kg', { locale: 'pt' }], + ['oitocentos kg', { locale: 'pt' }], + ['oitocentas kg', { locale: 'pt' }], + ['novecentos kg', { locale: 'pt' }], + ['novecentas kg', { locale: 'pt' }], + // bareScales + ['cem kg', { locale: 'pt' }], + ['mil kg', { locale: 'pt' }], + // tens + e + ones composition + ['vinte e cinco kg', { locale: 'pt' }], + ['trinta e sete kg', { locale: 'pt' }], + ['noventa e nove kg', { locale: 'pt' }], + // decimalWords + ['dois virgula cinco kg', { locale: 'pt' }], + ['dois virgula cinco seis kg', { locale: 'pt' }], + // approximatePhrases + ['mais ou menos 5 kg', { locale: 'pt' }], + ['por volta de 5 kg', { locale: 'pt' }], + ['cerca de 5 kg', { locale: 'pt' }], + ['la pelas 5 kg', { locale: 'pt' }], + // approximateWords additions + ['uns 5 kg', { locale: 'pt' }], + ['umas 3 kg', { locale: 'pt' }], + // trailingApproxPhrases + ['5 kg e pouco', { locale: 'pt' }], + ['5 kg e poucos', { locale: 'pt' }], + ['5 kg e tanto', { locale: 'pt' }], + // rangeAlternativeWords + ['5 ou 6 kg', { locale: 'pt' }], + // rangeFromWords 'de' + ['de 5 a 10 kg', { locale: 'pt' }], + // fuzzyAmounts + ['um par de kg', { locale: 'pt' }], + // Regional teen variants (PT-PT / BR) + ['dezessete kg', { locale: 'pt' }], + ['dezassete kg', { locale: 'pt' }], + ['dezanove kg', { locale: 'pt' }], + ['dezesseis kg', { locale: 'pt' }], + ['dezasseis kg', { locale: 'pt' }], +] + +export const dateRows = [ + // Pre-existing date behavior + ['ha tres dias', { locale: 'pt' }, 'relative-past-pin'], + ['amanha', { locale: 'pt' }, 'tomorrow-pin'], + ['ontem', { locale: 'pt' }, 'yesterday-pin'], + ['de madrugada', { locale: 'pt' }], + // dayOffsets: +2, -2 + ['anteontem', { locale: 'pt' }], + ['depois de amanha', { locale: 'pt' }], + // localized spoken clock + ['duas e meia', { locale: 'pt' }], + ['tres e quinze', { locale: 'pt' }], + ['quinze para as tres', { locale: 'pt' }], + ['dez para as oito', { locale: 'pt' }], + // localized period edges + ['inicio de julho', { locale: 'pt' }], + ['comeco de julho', { locale: 'pt' }], + ['no comeco do mes', { locale: 'pt' }], + ['meio de julho', { locale: 'pt' }], + ['fim de julho', { locale: 'pt' }], + ['final de julho', { locale: 'pt' }], +] + +export const dateRangeRows = [] + +export function buildContract({ lingo, localePacks, parseDate, parseDateRange }) { + return { + version: 1, + fixedNow: FIXED_NOW, + breadth: entriesFromRows(breadthRows, (input, opts) => lingo(input, opts)), + date: entriesFromRows( + dateRows, + (input, opts) => parseDate(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + dateRange: entriesFromRows( + dateRangeRows, + (input, opts) => parseDateRange(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + } +} + +function entriesFromRows(rows, parse, options = {}) { + const entries = {} + for (const row of rows) { + const [input, opts, label] = row + const key = uniqueKey(input, opts, label) + const parseOpts = options.defaultNow ? { now: FIXED_NOW, ...(opts ?? {}) } : opts + entries[key] = { + input, + ...(opts ? { opts } : {}), + ...summarize(parse(input, parseOpts)), + } + } + return entries +} + +function uniqueKey(input, opts, label) { + if (label) return `${input} [${label}]` + if (!opts?.now) return input + return `${input} [now:${opts.now.join(',')}]` +} + +function reviveDateOptions(opts, localePacks) { + if (!opts) return { ...(localePacks ? { localePacks } : {}), now: dateFromParts(FIXED_NOW) } + const out = { ...(localePacks ? { localePacks } : {}), ...opts } + const now = opts.now ?? FIXED_NOW + out.now = dateFromParts(now) + return out +} + +function summarize(result) { + const base = { + type: result.ok ? result.type : 'fail', + kind: null, + base: null, + unit: null, + issues: result.issues.map((issue) => issue.code), + span: result.ok ? { ...result.span } : null, + confidence: result.ok ? result.confidence : null, + } + if (!result.ok) return base + if (result.type === 'quantity') return { ...base, ...quantityShape(result.quantity) } + if (result.type === 'range') return { ...base, ...rangeShape(result.range) } + if (result.type === 'conversion') { + return { + ...base, + kind: result.converted.kind, + base: { + source: valueShape(result.source), + converted: valueShape(result.converted), + }, + unit: result.targetUnit, + } + } + if (result.type === 'number') { + return { ...base, base: number(result.value), unit: null } + } + if (result.type === 'date') { + return { + ...base, + kind: 'date', + base: dateParts(result.date), + unit: result.grain, + } + } + if (result.type === 'date-range') { + return { + ...base, + kind: 'date-range', + base: { + start: result.start ? dateParts(result.start.date) : null, + end: result.end ? dateParts(result.end.date) : null, + }, + unit: { + start: result.start ? result.start.grain : null, + end: result.end ? result.end.grain : null, + }, + } + } + return base +} + +function valueShape(value) { + return 'base' in value ? quantityShape(value) : rangeShape(value) +} + +function quantityShape(quantity) { + return { + kind: quantity.kind, + base: number(quantity.base), + unit: quantity.unit, + } +} + +function rangeShape(range) { + return { + kind: range.kind, + base: { + min: nullableNumber(range.minBase), + max: nullableNumber(range.maxBase), + ...(range.plusMinus + ? { + plusMinus: { + center: number(range.plusMinus.centerBase), + delta: number(range.plusMinus.deltaBase), + }, + } + : {}), + }, + unit: { + min: range.minUnit, + max: range.maxUnit, + ...(range.plusMinus ? { plusMinus: range.plusMinus.unit } : {}), + }, + } +} + +function dateFromParts(parts) { + return new Date(...parts) +} + +function dateParts(date) { + return { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + hour: date.getHours(), + minute: date.getMinutes(), + second: date.getSeconds(), + } +} + +function nullableNumber(value) { + return value === null ? null : number(value) +} + +function number(value) { + return Number(Number(value).toPrecision(15)) +} diff --git a/packages/lingo/tests/corpus/locale-zh-contract-v1.json b/packages/lingo/tests/corpus/locale-zh-contract-v1.json new file mode 100644 index 0000000..08ba954 --- /dev/null +++ b/packages/lingo/tests/corpus/locale-zh-contract-v1.json @@ -0,0 +1,599 @@ +{ + "version": 1, + "fixedNow": [ + 2026, + 6, + 3, + 14, + 30, + 0 + ], + "breadth": { + "5公斤 [kg-arabic-pin]": { + "input": "5公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "三公斤": { + "input": "三公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 3, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "十五公斤": { + "input": "十五公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 15, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "一百五十公斤": { + "input": "一百五十公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 150, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "一百五十米": { + "input": "一百五十米", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "length", + "base": 150, + "unit": "m", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "一百五": { + "input": "一百五", + "opts": { + "locale": "zh" + }, + "type": "number", + "kind": null, + "base": 150, + "unit": null, + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "三万五": { + "input": "三万五", + "opts": { + "locale": "zh" + }, + "type": "number", + "kind": null, + "base": 35000, + "unit": null, + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "3万5": { + "input": "3万5", + "opts": { + "locale": "zh" + }, + "type": "number", + "kind": null, + "base": 35000, + "unit": null, + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "3万5千公斤": { + "input": "3万5千公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 35000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "３５公斤": { + "input": "３５公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 35, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "两公斤半": { + "input": "两公斤半", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 2.5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "三个小时": { + "input": "三个小时", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "duration", + "base": 10800, + "unit": "h", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "三个月": { + "input": "三个月", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "duration", + "base": 7889400, + "unit": "mo", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "幺公斤": { + "input": "幺公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 1, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "差不多5公斤": { + "input": "差不多5公斤", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 3, + "end": 6 + }, + "confidence": 0.9 + }, + "三 或 四 公斤": { + "input": "三 或 四 公斤", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 3, + "max": 4 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 8 + }, + "confidence": 1 + }, + "七八天": { + "input": "七八天", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "duration", + "base": { + "min": 604800, + "max": 691200 + }, + "unit": { + "min": "d", + "max": "d" + }, + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + } + }, + "date": { + "三天前 [compact-days-ago-pin]": { + "input": "三天前", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 30, + "hour": 14, + "minute": 30, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "前天": { + "input": "前天", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "后天": { + "input": "后天", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "大前天": { + "input": "大前天", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 30, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "大后天": { + "input": "大后天", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "三个小时后": { + "input": "三个小时后", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 17, + "minute": 30, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "三天以后": { + "input": "三天以后", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 14, + "minute": 30, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "月底": { + "input": "月底", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "月初": { + "input": "月初", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "月中": { + "input": "月中", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 15, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "年底": { + "input": "年底", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 12, + "day": 31, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "年初": { + "input": "年初", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 1, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + }, + "周末": { + "input": "周末", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 1 + } + }, + "dateRange": {} +} diff --git a/packages/lingo/tests/corpus/locale-zh-source.mjs b/packages/lingo/tests/corpus/locale-zh-source.mjs new file mode 100644 index 0000000..0d0275f --- /dev/null +++ b/packages/lingo/tests/corpus/locale-zh-source.mjs @@ -0,0 +1,196 @@ +export const FIXED_NOW = [2026, 6, 3, 14, 30, 0] + +export const breadthRows = [ + ['5公斤', { locale: 'zh' }, 'kg-arabic-pin'], + ['三公斤', { locale: 'zh' }], + ['十五公斤', { locale: 'zh' }], + ['一百五十公斤', { locale: 'zh' }], + ['一百五十米', { locale: 'zh' }], + ['一百五', { locale: 'zh' }], + ['三万五', { locale: 'zh' }], + ['3万5', { locale: 'zh' }], + ['3万5千公斤', { locale: 'zh' }], + ['３５公斤', { locale: 'zh' }], + ['两公斤半', { locale: 'zh' }], + ['三个小时', { locale: 'zh' }], + ['三个月', { locale: 'zh' }], + ['幺公斤', { locale: 'zh' }], + ['差不多5公斤', { locale: 'zh' }], + ['三 或 四 公斤', { locale: 'zh' }], + ['七八天', { locale: 'zh' }], +] + +export const dateRows = [ + ['三天前', { locale: 'zh' }, 'compact-days-ago-pin'], + ['前天', { locale: 'zh' }], + ['后天', { locale: 'zh' }], + ['大前天', { locale: 'zh' }], + ['大后天', { locale: 'zh' }], + ['三个小时后', { locale: 'zh' }], + ['三天以后', { locale: 'zh' }], + ['月底', { locale: 'zh' }], + ['月初', { locale: 'zh' }], + ['月中', { locale: 'zh' }], + ['年底', { locale: 'zh' }], + ['年初', { locale: 'zh' }], + ['周末', { locale: 'zh' }], +] + +export const dateRangeRows = [] + +export function buildContract({ lingo, localePacks, parseDate, parseDateRange }) { + return { + version: 1, + fixedNow: FIXED_NOW, + breadth: entriesFromRows(breadthRows, (input, opts) => lingo(input, opts)), + date: entriesFromRows( + dateRows, + (input, opts) => parseDate(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + dateRange: entriesFromRows( + dateRangeRows, + (input, opts) => parseDateRange(input, reviveDateOptions(opts, localePacks)), + { defaultNow: true }, + ), + } +} + +function entriesFromRows(rows, parse, options = {}) { + const entries = {} + for (const row of rows) { + const [input, opts, label] = row + const key = uniqueKey(input, opts, label) + const parseOpts = options.defaultNow ? { now: FIXED_NOW, ...(opts ?? {}) } : opts + entries[key] = { + input, + ...(opts ? { opts } : {}), + ...summarize(parse(input, parseOpts)), + } + } + return entries +} + +function uniqueKey(input, opts, label) { + if (label) return `${input} [${label}]` + if (!opts?.now) return input + return `${input} [now:${opts.now.join(',')}]` +} + +function reviveDateOptions(opts, localePacks) { + if (!opts) return { ...(localePacks ? { localePacks } : {}), now: dateFromParts(FIXED_NOW) } + const out = { ...(localePacks ? { localePacks } : {}), ...opts } + const now = opts.now ?? FIXED_NOW + out.now = dateFromParts(now) + return out +} + +function summarize(result) { + const base = { + type: result.ok ? result.type : 'fail', + kind: null, + base: null, + unit: null, + issues: result.issues.map((issue) => issue.code), + span: result.ok ? { ...result.span } : null, + confidence: result.ok ? result.confidence : null, + } + if (!result.ok) return base + if (result.type === 'quantity') return { ...base, ...quantityShape(result.quantity) } + if (result.type === 'range') return { ...base, ...rangeShape(result.range) } + if (result.type === 'conversion') { + return { + ...base, + kind: result.converted.kind, + base: { + source: valueShape(result.source), + converted: valueShape(result.converted), + }, + unit: result.targetUnit, + } + } + if (result.type === 'number') { + return { ...base, base: number(result.value), unit: null } + } + if (result.type === 'date') { + return { + ...base, + kind: 'date', + base: dateParts(result.date), + unit: result.grain, + } + } + if (result.type === 'date-range') { + return { + ...base, + kind: 'date-range', + base: { + start: result.start ? dateParts(result.start.date) : null, + end: result.end ? dateParts(result.end.date) : null, + }, + unit: { + start: result.start ? result.start.grain : null, + end: result.end ? result.end.grain : null, + }, + } + } + return base +} + +function valueShape(value) { + return 'base' in value ? quantityShape(value) : rangeShape(value) +} + +function quantityShape(quantity) { + return { + kind: quantity.kind, + base: number(quantity.base), + unit: quantity.unit, + } +} + +function rangeShape(range) { + return { + kind: range.kind, + base: { + min: nullableNumber(range.minBase), + max: nullableNumber(range.maxBase), + ...(range.plusMinus + ? { + plusMinus: { + center: number(range.plusMinus.centerBase), + delta: number(range.plusMinus.deltaBase), + }, + } + : {}), + }, + unit: { + min: range.minUnit, + max: range.maxUnit, + ...(range.plusMinus ? { plusMinus: range.plusMinus.unit } : {}), + }, + } +} + +function dateFromParts(parts) { + return new Date(...parts) +} + +function dateParts(date) { + return { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + hour: date.getHours(), + minute: date.getMinutes(), + second: date.getSeconds(), + } +} + +function nullableNumber(value) { + return value === null ? null : number(value) +} + +function number(value) { + return Number(Number(value).toPrecision(15)) +} diff --git a/packages/lingo/tests/corpus/source.mjs b/packages/lingo/tests/corpus/source.mjs index 5fa827e..07a391c 100644 --- a/packages/lingo/tests/corpus/source.mjs +++ b/packages/lingo/tests/corpus/source.mjs @@ -188,6 +188,8 @@ export const breadthRows = [ ['a dozen inches'], ['half a dozen feet'], ['one hundred and five kg'], + ['two point five six kg'], + ['three point one four kg'], [''], [' '], ['banana'], diff --git a/packages/lingo/tests/locales/cjk.test.ts b/packages/lingo/tests/locales/cjk.test.ts index 19e4789..b803a11 100644 --- a/packages/lingo/tests/locales/cjk.test.ts +++ b/packages/lingo/tests/locales/cjk.test.ts @@ -56,6 +56,137 @@ describe('CJK locale packs', () => { } }) + it('sub-segments Chinese CJK number/unit tokens with original spans', () => { + const lingo = createLingo({ locales: [zh] }) + const cases = [ + ['三公斤', 'kg', 3], + ['十五公斤', 'kg', 15], + ['一百五十公斤', 'kg', 150], + ['一百五十米', 'm', 150], + ] as const + + for (const [input, unit, value] of cases) { + const result = lingo.parseQuantity(input, { locale: 'zh' }) + expect(result.ok, `${input}: ${JSON.stringify(result.issues)}`).toBe(true) + if (!result.ok) { + continue + } + expect(result.locale).toBe('zh') + expect(result.quantity.unit).toBe(unit) + expect(result.quantity.value).toBe(value) + expect(result.span).toEqual({ start: 0, end: input.length }) + } + }) + + it('parses mixed Arabic/CJK Chinese scales without enabling Latin glued multipliers', () => { + const lingo = createLingo({ locales: [zh] }) + + const bare = lingo.parse('3万', { locale: 'zh' }) + expect(bare.ok && bare.type === 'number' ? bare.value : Number.NaN).toBe(30_000) + if (bare.ok) { + expect(bare.span).toEqual({ start: 0, end: '3万'.length }) + } + + const quantity = lingo.parseQuantity('3万5千公斤', { locale: 'zh' }) + expect(quantity.ok, JSON.stringify(quantity.issues)).toBe(true) + if (quantity.ok) { + expect(quantity.quantity.unit).toBe('kg') + expect(quantity.quantity.value).toBe(35_000) + expect(quantity.span).toEqual({ start: 0, end: '3万5千公斤'.length }) + } + + const tenMillion = lingo.parse('千万', { locale: 'zh' }) + expect(tenMillion.ok && tenMillion.type === 'number' ? tenMillion.value : Number.NaN).toBe( + 10_000_000, + ) + + expect(lingo.parse('万万', { locale: 'zh' }).ok).toBe(false) + expect(lingo.parse('亿万', { locale: 'zh' }).ok).toBe(false) + + const latin = lingo.parseQuantity('1M') + expect(latin.ok).toBe(false) + }) + + it('parses Chinese adjacent-number ranges and post-unit half', () => { + const lingo = createLingo({ locales: [zh] }) + + const range = lingo.parseRange('七八天', { locale: 'zh' }) + expect(range.ok, JSON.stringify(range.issues)).toBe(true) + if (range.ok) { + expect(range.range.min()?.value).toBe(7) + expect(range.range.max()?.value).toBe(8) + expect(range.range.min()?.unit).toBe('d') + expect(range.span).toEqual({ start: 0, end: '七八天'.length }) + } + + const repeated = lingo.parse('三三天', { locale: 'zh' }) + expect(repeated.ok).toBe(false) + + const half = lingo.parseQuantity('两公斤半', { locale: 'zh' }) + expect(half.ok, JSON.stringify(half.issues)).toBe(true) + if (half.ok) { + expect(half.quantity.unit).toBe('kg') + expect(half.quantity.value).toBe(2.5) + expect(half.span).toEqual({ start: 0, end: '两公斤半'.length }) + } + }) + + it('parses deepened Chinese data-only idioms', () => { + const lingo = createLingo({ locales: [zh] }) + + const fullWidth = lingo.parseQuantity('３５公斤', { locale: 'zh' }) + expect(fullWidth.ok, JSON.stringify(fullWidth.issues)).toBe(true) + if (fullWidth.ok) { + expect(fullWidth.quantity.unit).toBe('kg') + expect(fullWidth.quantity.value).toBe(35) + expect(fullWidth.span).toEqual({ start: 0, end: '３５公斤'.length }) + } + + const classifierDuration = lingo.parseQuantity('三个小时', { locale: 'zh' }) + expect(classifierDuration.ok, JSON.stringify(classifierDuration.issues)).toBe(true) + if (classifierDuration.ok) { + expect(classifierDuration.quantity.unit).toBe('h') + expect(classifierDuration.quantity.value).toBe(3) + expect(classifierDuration.span).toEqual({ start: 0, end: '三个小时'.length }) + } + + const approximate = lingo.parseQuantity('差不多5公斤', { locale: 'zh' }) + expect(approximate.ok, JSON.stringify(approximate.issues)).toBe(true) + if (approximate.ok) { + expect(approximate.quantity.unit).toBe('kg') + expect(approximate.quantity.value).toBe(5) + expect(approximate.quantity.approximate).toBe(true) + } + + const alternative = lingo.parseRange('三 或 四 公斤', { locale: 'zh' }) + expect(alternative.ok, JSON.stringify(alternative.issues)).toBe(true) + if (alternative.ok) { + expect(alternative.range.min()?.value).toBe(3) + expect(alternative.range.max()?.value).toBe(4) + expect(alternative.range.min()?.unit).toBe('kg') + } + }) + + it('parses Chinese day offsets and classifier duration offsets', () => { + const cases = [ + ['前天', new Date(2026, 6, 6)], + ['后天', new Date(2026, 6, 10)], + ['大前天', new Date(2026, 6, 5)], + ['大后天', new Date(2026, 6, 11)], + ['三个小时后', new Date(2026, 6, 8, 15)], + ['三天以后', new Date(2026, 6, 11, 12)], + ] as const + + for (const [input, expected] of cases) { + const result = parseDate(input, { now: NOW, locale: 'zh', localePacks: [zh] }) + expect(result.ok, `${input}: ${JSON.stringify(result.issues)}`).toBe(true) + if (!result.ok) { + continue + } + expect(result.date).toEqual(expected) + } + }) + it('parses Japanese compact quantities, dates, and fuzzy temperature', () => { const lingo = createLingo({ locales: [ja] }) @@ -99,6 +230,148 @@ describe('CJK locale packs', () => { } }) + it('sub-segments Japanese CJK number/unit tokens with original spans', () => { + const lingo = createLingo({ locales: [ja] }) + const cases = [ + ['三キロ', 'kg', 3], + ['三十五キロ', 'kg', 35], + ['百五十グラム', 'g', 150], + ['３５キロ', 'kg', 35], + ] as const + + for (const [input, unit, value] of cases) { + const result = lingo.parseQuantity(input, { locale: 'ja' }) + expect(result.ok, `${input}: ${JSON.stringify(result.issues)}`).toBe(true) + if (!result.ok) { + continue + } + expect(result.locale).toBe('ja') + expect(result.quantity.unit).toBe(unit) + expect(result.quantity.value).toBe(value) + expect(result.span).toEqual({ start: 0, end: input.length }) + } + }) + + it('parses Japanese mixed CJK scales, wave-dash ranges, and post-unit half', () => { + const lingo = createLingo({ locales: [ja] }) + + const mixed = lingo.parseQuantity('3万5千キロ', { locale: 'ja' }) + expect(mixed.ok, JSON.stringify(mixed.issues)).toBe(true) + if (mixed.ok) { + expect(mixed.quantity.unit).toBe('kg') + expect(mixed.quantity.value).toBe(35_000) + expect(mixed.span).toEqual({ start: 0, end: '3万5千キロ'.length }) + } + + const large = lingo.parse('1億2千万', { locale: 'ja' }) + expect(large.ok && large.type === 'number' ? large.value : Number.NaN).toBe(120_000_000) + if (large.ok) { + expect(large.span).toEqual({ start: 0, end: '1億2千万'.length }) + } + + const kanjiLarge = lingo.parse('一億二千万', { locale: 'ja' }) + expect(kanjiLarge.ok && kanjiLarge.type === 'number' ? kanjiLarge.value : Number.NaN).toBe( + 120_000_000, + ) + + for (const input of ['5〜10キロ', '5～10キロ'] as const) { + const range = lingo.parseRange(input, { locale: 'ja' }) + expect(range.ok, `${input}: ${JSON.stringify(range.issues)}`).toBe(true) + if (!range.ok) { + continue + } + expect(range.range.min()?.value).toBe(5) + expect(range.range.max()?.value).toBe(10) + expect(range.range.min()?.unit).toBe('kg') + expect(range.span).toEqual({ start: 0, end: input.length }) + } + + const dayRange = lingo.parseRange('三〜五日', { locale: 'ja' }) + expect(dayRange.ok, JSON.stringify(dayRange.issues)).toBe(true) + if (dayRange.ok) { + expect(dayRange.range.min()?.value).toBe(3) + expect(dayRange.range.max()?.value).toBe(5) + expect(dayRange.range.min()?.unit).toBe('d') + expect(dayRange.span).toEqual({ start: 0, end: '三〜五日'.length }) + } + + const massHalf = lingo.parseQuantity('二キロ半', { locale: 'ja' }) + expect(massHalf.ok, JSON.stringify(massHalf.issues)).toBe(true) + if (massHalf.ok) { + expect(massHalf.quantity.unit).toBe('kg') + expect(massHalf.quantity.value).toBe(2.5) + expect(massHalf.span).toEqual({ start: 0, end: '二キロ半'.length }) + } + + const durationHalf = lingo.parseQuantity('一時間半', { locale: 'ja' }) + expect(durationHalf.ok, JSON.stringify(durationHalf.issues)).toBe(true) + if (durationHalf.ok) { + expect(durationHalf.quantity.unit).toBe('h') + expect(durationHalf.quantity.value).toBe(1.5) + expect(durationHalf.span).toEqual({ start: 0, end: '一時間半'.length }) + } + }) + + it('parses deepened Japanese data-only idioms', () => { + const lingo = createLingo({ locales: [ja] }) + + const yen = lingo.parseQuantity('三万五千円', { locale: 'ja' }) + expect(yen.ok, JSON.stringify(yen.issues)).toBe(true) + if (yen.ok) { + expect(yen.quantity.kind).toBe('currency') + expect(yen.quantity.unit).toBe('JPY') + expect(yen.quantity.value).toBe(35_000) + expect(yen.span).toEqual({ start: 0, end: '三万五千円'.length }) + } + + const largeYen = lingo.parseQuantity('二億三千万円', { locale: 'ja' }) + expect(largeYen.ok, JSON.stringify(largeYen.issues)).toBe(true) + if (largeYen.ok) { + expect(largeYen.quantity.unit).toBe('JPY') + expect(largeYen.quantity.value).toBe(230_000_000) + expect(largeYen.span).toEqual({ start: 0, end: '二億三千万円'.length }) + } + + const approximate = lingo.parseQuantity('5キロ ほど', { locale: 'ja' }) + expect(approximate.ok, JSON.stringify(approximate.issues)).toBe(true) + if (approximate.ok) { + expect(approximate.quantity.unit).toBe('kg') + expect(approximate.quantity.value).toBe(5) + expect(approximate.quantity.approximate).toBe(true) + } + + const counter = lingo.parse('三個', { locale: 'ja' }) + expect(counter.ok, JSON.stringify(counter.issues)).toBe(true) + if (counter.ok) { + expect(counter.type).toBe('number') + if (counter.type === 'number') { + expect(counter.value).toBe(3) + } + } + }) + + it('parses Japanese day offsets and day-time phrases', () => { + const cases = [ + ['一昨日', new Date(2026, 6, 6)], + ['おととい', new Date(2026, 6, 6)], + ['明後日', new Date(2026, 6, 10)], + ['あさって', new Date(2026, 6, 10)], + ['しあさって', new Date(2026, 6, 11)], + ['今朝', new Date(2026, 6, 8, 8)], + ['今晩', new Date(2026, 6, 8, 19)], + ['今夜', new Date(2026, 6, 8, 21)], + ] as const + + for (const [input, expected] of cases) { + const result = parseDate(input, { now: NOW, locale: 'ja', localePacks: [ja] }) + expect(result.ok, `${input}: ${JSON.stringify(result.issues)}`).toBe(true) + if (!result.ok) { + continue + } + expect(result.date).toEqual(expected) + } + }) + it('pins bare CJK noon to today like English "noon"', () => { // An afternoon `now`: an unpinned noon would forward-roll to tomorrow. const afternoon = new Date(2026, 6, 8, 15) diff --git a/packages/lingo/tests/locales/romance.test.ts b/packages/lingo/tests/locales/romance.test.ts index b1b936b..96a9f1b 100644 --- a/packages/lingo/tests/locales/romance.test.ts +++ b/packages/lingo/tests/locales/romance.test.ts @@ -7,6 +7,7 @@ import { fr } from '../../src/locales/fr' import { pt } from '../../src/locales/pt' const NOW = new Date(2026, 6, 8, 12) +const CLOCK_NOW = new Date(2026, 6, 8, 1) function expectQuantity(input: string, locale: 'es' | 'fr' | 'pt', value: number) { const packs = { es, fr, pt } @@ -34,6 +35,20 @@ function expectRange(input: string, locale: 'es' | 'fr' | 'pt', min: number, max } } +function expectNumber(input: string, locale: 'es' | 'fr' | 'pt', value: number) { + const packs = { es, fr, pt } + const lingo = createLingo({ locales: [packs[locale]] }) + const result = lingo.parse(input, { locale }) + expect(result.ok, JSON.stringify(result.issues)).toBe(true) + if (result.ok) { + expect(result.locale).toBe(locale) + expect(result.type).toBe('number') + if (result.type === 'number') { + expect(result.value).toBe(value) + } + } +} + function expectDate(input: string, locale: 'es' | 'fr' | 'pt', day: number) { const packs = { es, fr, pt } const result = parseDate(input, { now: NOW, locale, localePacks: [packs[locale]] }) @@ -51,9 +66,10 @@ function expectDateTime( locale: 'es' | 'fr' | 'pt', expected: Date, grain: DateGrain = 'day', + now: Date = NOW, ) { const packs = { es, fr, pt } - const result = parseDate(input, { now: NOW, locale, localePacks: [packs[locale]] }) + const result = parseDate(input, { now, locale, localePacks: [packs[locale]] }) expect(result.ok, `${input}: ${JSON.stringify(result.issues)}`).toBe(true) if (!result.ok) { return @@ -63,7 +79,7 @@ function expectDateTime( expect(result.span).toEqual({ start: 0, end: input.length }) const canonical = parseDate(dayIso(result.date), { - now: NOW, + now, locale, localePacks: [packs[locale]], }) @@ -98,10 +114,17 @@ describe('Romance locale packs', () => { expectDateTime('mañana', 'es', new Date(2026, 6, 9), 'day') expectDateTime('mañana por la mañana', 'es', new Date(2026, 6, 9, 9), 'hour') expectDateTime('mediodía mañana', 'es', new Date(2026, 6, 9, 12), 'hour') + expectDateTime('las dos y media', 'es', new Date(2026, 6, 8, 2, 30), 'minute', CLOCK_NOW) + expectDateTime('las tres menos cuarto', 'es', new Date(2026, 6, 8, 2, 45), 'minute', CLOCK_NOW) + expectDateTime('a principios de mes', 'es', new Date(2026, 6, 1), 'day') + expectDateTime('a mediados de julio', 'es', new Date(2026, 6, 15), 'day') + expectDateTime('a finales de mes', 'es', new Date(2026, 6, 31), 'day') expectDateTime('el mes que viene', 'es', new Date(2026, 7, 1), 'month') expectDateTime('el próximo mes', 'es', new Date(2026, 7, 1), 'month') expectDateTime('12 de julio de 2026', 'es', new Date(2026, 6, 12), 'day') expectDateTime('hace 3 días', 'es', new Date(2026, 6, 5, 12), 'day') + expectRange('entre 5 y 10 kg', 'es', 5, 10) + expectNumber('treinta y cinco', 'es', 35) }) it('parses French quantities, ranges, and relative dates', () => { @@ -115,10 +138,21 @@ describe('Romance locale packs', () => { expectDateTime('midi demain', 'fr', new Date(2026, 6, 9, 12), 'hour') expectDateTime('demain matin', 'fr', new Date(2026, 6, 9, 9), 'hour') expectDateTime('demain', 'fr', new Date(2026, 6, 9), 'day') + expectDateTime('deux heures et quart', 'fr', new Date(2026, 6, 8, 2, 15), 'minute', CLOCK_NOW) + expectDateTime( + 'trois heures moins le quart', + 'fr', + new Date(2026, 6, 8, 2, 45), + 'minute', + CLOCK_NOW, + ) expectDateTime('le mois prochain', 'fr', new Date(2026, 7, 1), 'month') expectDateTime('dans 3 jours', 'fr', new Date(2026, 6, 11, 12), 'day') expectDateTime('12 juillet 2026', 'fr', new Date(2026, 6, 12), 'day') expectDateTime('juillet dernier', 'fr', new Date(2025, 6, 1), 'month') + expectDateTime('mi-juillet', 'fr', new Date(2026, 6, 15), 'day') + expectDateTime('fin juillet', 'fr', new Date(2026, 6, 31), 'day') + expectDateTime('lundi en huit', 'fr', new Date(2026, 6, 20), 'day') }) it('parses Portuguese quantities, ranges, and relative dates', () => { @@ -131,8 +165,13 @@ describe('Romance locale packs', () => { it('parses Portuguese date/time phrases', () => { expectDateTime('amanhã de manhã', 'pt', new Date(2026, 6, 9, 9), 'hour') expectDateTime('meio-dia amanhã', 'pt', new Date(2026, 6, 9, 12), 'hour') + expectDateTime('duas e meia', 'pt', new Date(2026, 6, 8, 2, 30), 'minute', CLOCK_NOW) + expectDateTime('quinze para as tres', 'pt', new Date(2026, 6, 8, 2, 45), 'minute', CLOCK_NOW) expectDateTime('mês que vem', 'pt', new Date(2026, 7, 1), 'month') expectDateTime('12 de julho', 'pt', new Date(2026, 6, 12), 'day') + expectDateTime('meio de julho', 'pt', new Date(2026, 6, 15), 'day') + expectDateTime('fim de julho', 'pt', new Date(2026, 6, 31), 'day') + expectDateTime('no começo do mês', 'pt', new Date(2026, 6, 1), 'day') }) it('pins bare midday to today like English "noon"; midnight still rolls forward', () => { @@ -154,6 +193,338 @@ describe('Romance locale packs', () => { }) }) +// ─── Plan 033: locale idiom coverage ────────────────────────────────────────── + +function expectApprox(input: string, locale: 'es' | 'fr' | 'pt', value: number) { + const packs = { es, fr, pt } + const lingo = createLingo({ locales: [packs[locale]] }) + const result = lingo.parseQuantity(input, { locale }) + expect(result.ok, `${input}: ${JSON.stringify(result.issues)}`).toBe(true) + if (result.ok) { + expect(result.quantity.value).toBe(value) + expect(result.quantity.approximate).toBe(true) + } +} + +function expectFuzzy(input: string, locale: 'es' | 'fr' | 'pt', value: number) { + const packs = { es, fr, pt } + const lingo = createLingo({ locales: [packs[locale]] }) + const result = lingo.parse(input, { locale }) + expect(result.ok, `${input}: ${JSON.stringify(result.issues)}`).toBe(true) + if (result.ok) { + expect(result.type, `${input}: expected range for fuzzy`).toBe('range') + if (result.type === 'range') { + expect(result.range.min()?.value, `${input}: min`).toBeLessThanOrEqual(value) + expect(result.range.max()?.value, `${input}: max`).toBeGreaterThanOrEqual(value) + expect(result.range.approximate).toBe(true) + } + } +} + +describe('Plan 033 — ES idioms', () => { + it('composed: veintiuno-veintinueve fused forms', () => { + expectQuantity('veintiuno kg', 'es', 21) + expectQuantity('veintiún kg', 'es', 21) + expectQuantity('veinticinco kg', 'es', 25) + expectQuantity('veintinueve kg', 'es', 29) + }) + + it('composed: compound hundreds (doscientos-novecientos)', () => { + expectQuantity('doscientos kg', 'es', 200) + expectQuantity('quinientos kg', 'es', 500) + expectQuantity('novecientos kg', 'es', 900) + expectQuantity('doscientas kg', 'es', 200) + expectQuantity('quinientas kg', 'es', 500) + }) + + it('bareScales: cien/mil without preceding multiplier', () => { + const lingo = createLingo({ locales: [es] }) + const r = lingo.parseQuantity('cien kg', { locale: 'es' }) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + } + + const r2 = lingo.parseQuantity('mil kg', { locale: 'es' }) + expect(r2.ok).toBe(true) + if (r2.ok) { + expect(r2.quantity.value).toBe(1000) + } + }) + + it('bareScale + composed: mil quinientos = 1500', () => { + expectQuantity('mil quinientos kg', 'es', 1500) + }) + + it('decimalWords: dos coma cinco = 2.5', () => { + const lingo = createLingo({ locales: [es] }) + const r = lingo.parseQuantity('dos coma cinco kg', { locale: 'es' }) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) + + it('approximatePhrases: mas o menos 5 kilos', () => { + expectApprox('mas o menos 5 kilos', 'es', 5) + expectApprox('cosa de 5 kilos', 'es', 5) + }) + + it('approximateWords: unos/unas', () => { + expectApprox('unos 5 kilos', 'es', 5) + expectApprox('unas 3 kilos', 'es', 3) + }) + + it('trailingApproxPhrases: y pico, y tantos, y algo', () => { + expectApprox('5 kilos y pico', 'es', 5) + expectApprox('5 kilos y tantos', 'es', 5) + }) + + it('rangeAlternativeWords: o', () => { + expectRange('5 o 6 kg', 'es', 5, 6) + }) + + it('rangeFromWords: de 5 a 10 kg', () => { + expectRange('de 5 a 10 kg', 'es', 5, 10) + }) + + it('fuzzyAmounts: un par de kilos', () => { + expectFuzzy('un par de kilos', 'es', 2) + }) + + it('dayOffsets: pasado manana (+2), anteayer/antier (-2)', () => { + const r1 = parseDate('pasado mañana', { now: NOW, locale: 'es', localePacks: [es] }) + expect(r1.ok).toBe(true) + if (r1.ok) { + expect(r1.date.getDate()).toBe(10) + } + + const r2 = parseDate('anteayer', { now: NOW, locale: 'es', localePacks: [es] }) + expect(r2.ok).toBe(true) + if (r2.ok) { + expect(r2.date.getDate()).toBe(6) + } + + const r3 = parseDate('antier', { now: NOW, locale: 'es', localePacks: [es] }) + expect(r3.ok).toBe(true) + if (r3.ok) { + expect(r3.date.getDate()).toBe(6) + } + }) +}) + +describe('Plan 033 — FR idioms', () => { + it('composed: vigesimal soixante-dix = 70', () => { + expectQuantity('soixante-dix kg', 'fr', 70) + }) + + it('composed: vigesimal soixante-quinze = 75', () => { + expectQuantity('soixante-quinze kg', 'fr', 75) + }) + + it('composed: quatre-vingts = 80', () => { + expectQuantity('quatre-vingts kg', 'fr', 80) + }) + + it('composed: quatre-vingt-dix = 90', () => { + expectQuantity('quatre-vingt-dix kg', 'fr', 90) + }) + + it('composed: quatre-vingt-dix-neuf = 99', () => { + expectQuantity('quatre-vingt-dix-neuf kg', 'fr', 99) + }) + + it('bareScales: cent/mille without preceding multiplier', () => { + const lingo = createLingo({ locales: [fr] }) + const r = lingo.parseQuantity('cent kg', { locale: 'fr' }) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + } + + const r2 = lingo.parseQuantity('mille kg', { locale: 'fr' }) + expect(r2.ok).toBe(true) + if (r2.ok) { + expect(r2.quantity.value).toBe(1000) + } + }) + + it('decimalWords: deux virgule cinq = 2.5', () => { + const lingo = createLingo({ locales: [fr] }) + const r = lingo.parseQuantity('deux virgule cinq kg', { locale: 'fr' }) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) + + it('approximatePhrases: a peu pres 5 kilos', () => { + expectApprox('a peu pres 5 kilos', 'fr', 5) + expectApprox('grosso modo 5 kilos', 'fr', 5) + }) + + it('trailingApproxWords: environ (trailing)', () => { + expectApprox('5 kilos environ', 'fr', 5) + }) + + it('trailingApproxPhrases: a peu pres / et quelques', () => { + expectApprox('5 kilos a peu pres', 'fr', 5) + expectApprox('5 kg et quelques', 'fr', 5) + }) + + it('rangeAlternativeWords: ou', () => { + expectRange('5 ou 6 kg', 'fr', 5, 6) + }) + + it('fuzzyAmounts: une dizaine de kilos', () => { + expectFuzzy('une dizaine de kilos', 'fr', 10) + expectFuzzy('une vingtaine de kilos', 'fr', 20) + expectFuzzy('une trentaine de kilos', 'fr', 30) + expectFuzzy('une centaine de kilos', 'fr', 100) + expectFuzzy('un millier de kilos', 'fr', 1000) + expectFuzzy('quelques kilos', 'fr', 3) + }) + + it('dayOffsets: apres-demain (+2), avant-hier (-2)', () => { + const r1 = parseDate('après-demain', { now: NOW, locale: 'fr', localePacks: [fr] }) + expect(r1.ok).toBe(true) + if (r1.ok) { + expect(r1.date.getDate()).toBe(10) + } + + const r2 = parseDate('avant-hier', { now: NOW, locale: 'fr', localePacks: [fr] }) + expect(r2.ok).toBe(true) + if (r2.ok) { + expect(r2.date.getDate()).toBe(6) + } + }) +}) + +describe('Plan 033 — PT idioms', () => { + it('composed: compound hundreds duzentos-novecentos', () => { + expectQuantity('duzentos kg', 'pt', 200) + expectQuantity('quatrocentas kg', 'pt', 400) + expectQuantity('quinhentos kg', 'pt', 500) + expectQuantity('seiscentas kg', 'pt', 600) + expectQuantity('setecentas kg', 'pt', 700) + expectQuantity('oitocentas kg', 'pt', 800) + expectQuantity('novecentos kg', 'pt', 900) + expectQuantity('duzentas kg', 'pt', 200) + expectQuantity('quinhentas kg', 'pt', 500) + }) + + it('bareScales: cem/mil without preceding multiplier', () => { + const lingo = createLingo({ locales: [pt] }) + const r = lingo.parseQuantity('cem kg', { locale: 'pt' }) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(100) + } + + const r2 = lingo.parseQuantity('mil kg', { locale: 'pt' }) + expect(r2.ok).toBe(true) + if (r2.ok) { + expect(r2.quantity.value).toBe(1000) + } + }) + + it('decimalWords: dois virgula cinco = 2.5', () => { + const lingo = createLingo({ locales: [pt] }) + const r = lingo.parseQuantity('dois virgula cinco kg', { locale: 'pt' }) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(2.5) + } + }) + + it('approximatePhrases: mais ou menos 5 quilos', () => { + expectApprox('mais ou menos 5 quilos', 'pt', 5) + expectApprox('por volta de 5 quilos', 'pt', 5) + expectApprox('la pelas 5 quilos', 'pt', 5) + }) + + it('approximateWords: uns/umas', () => { + expectApprox('uns 5 quilos', 'pt', 5) + expectApprox('umas 3 quilos', 'pt', 3) + }) + + it('trailingApproxPhrases: e pouco/e poucos/e tanto', () => { + expectApprox('5 quilos e pouco', 'pt', 5) + expectApprox('5 quilos e poucos', 'pt', 5) + expectApprox('5 quilos e tanto', 'pt', 5) + }) + + it('rangeAlternativeWords: ou', () => { + expectRange('5 ou 6 kg', 'pt', 5, 6) + }) + + it('rangeFromWords: de 5 a 10 kg', () => { + expectRange('de 5 a 10 kg', 'pt', 5, 10) + }) + + it('fuzzyAmounts: um par de quilos', () => { + expectFuzzy('um par de quilos', 'pt', 2) + }) + + it('regional variants: dezessete (BR) and dezassete (PT)', () => { + expectQuantity('dezessete kg', 'pt', 17) + expectQuantity('dezassete kg', 'pt', 17) + expectQuantity('dezanove kg', 'pt', 19) + }) + + it('dayOffsets: anteontem (-2), depois de amanha (+2)', () => { + const r1 = parseDate('anteontem', { now: NOW, locale: 'pt', localePacks: [pt] }) + expect(r1.ok).toBe(true) + if (r1.ok) { + expect(r1.date.getDate()).toBe(6) + } + + const r2 = parseDate('depois de amanhã', { now: NOW, locale: 'pt', localePacks: [pt] }) + expect(r2.ok).toBe(true) + if (r2.ok) { + expect(r2.date.getDate()).toBe(10) + } + }) + + it('dayTimePhrases: de madrugada', () => { + expectDateTime('de madrugada', 'pt', new Date(2026, 6, 9, 4), 'hour') + }) +}) + +// ─── F3: auto-detection scores composed/bareScales/decimalWords ───────────── + +describe('F3: auto-detection with composed/bareScale words', () => { + it('"quinientos kg" auto-detects as es', () => { + const lingo = createLingo({ locales: [es, fr, pt] }) + const r = lingo.parseQuantity('quinientos kg') + expect(r.ok, JSON.stringify(r.issues)).toBe(true) + if (r.ok) { + expect(r.locale).toBe('es') + expect(r.quantity.value).toBe(500) + } + }) + + it('"quatre-vingt-dix kg" auto-detects as fr', () => { + const lingo = createLingo({ locales: [es, fr, pt] }) + const r = lingo.parseQuantity('quatre-vingt-dix kg') + expect(r.ok, JSON.stringify(r.issues)).toBe(true) + if (r.ok) { + expect(r.locale).toBe('fr') + expect(r.quantity.value).toBe(90) + } + }) + + it('"duzentos kg" auto-detects as pt', () => { + const lingo = createLingo({ locales: [es, fr, pt] }) + const r = lingo.parseQuantity('duzentos kg') + expect(r.ok, JSON.stringify(r.issues)).toBe(true) + if (r.ok) { + expect(r.locale).toBe('pt') + expect(r.quantity.value).toBe(200) + } + }) +}) + function dayIso(date: Date): string { return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')}` } diff --git a/plans/033-locale-idiom-coverage.md b/plans/033-locale-idiom-coverage.md new file mode 100644 index 0000000..a195e97 --- /dev/null +++ b/plans/033-locale-idiom-coverage.md @@ -0,0 +1,82 @@ +--- +id: 033 +title: Locale idiom coverage +status: in-progress +created: 2026-07-09 +updated: 2026-07-09 +--- + +# Locale idiom coverage — parse how people actually write and speak + +Plan 031 shipped the locale-pack *infrastructure*; this plan makes the packs +*comprehensive*: the everyday spoken/written idioms natives actually use for +quantities, dates, and amounts, in every shipped language. Research pass +2026-07-09 (six native-level idiom studies + prior-art + engine-gap audit; +distilled in `wiki/research/locale-idioms.md`) produced ~335 test-precise gap +idioms and identified the engine seams that block whole idiom classes +regardless of pack data. + +## Strategy + +1. **Idiom-first, corpus-gated.** Every idiom lands as a corpus row before or + with its implementation: exact native input (real diacritics/script) → + exact canonical value. Per-locale corpus files + (`tests/corpus/locale--source.mjs` → `locale-.json` contracts) + mirror the main corpus mechanics and gate `bun run check`, so locale + behavior can never silently regress. Frequency tags (very-common/common/ + occasional) rank what ships first. +2. **Engine stays language-neutral; packs stay data.** Idiom classes blocked + by the engine get the *smallest data-driven extension*: a new optional + pack field consumed by existing grammar (D5). No per-language parser + branches (the chrono-node lesson: 14 imperative locale parsers still miss + vigesimal French and "en huit" — imperative locale code doesn't scale). +3. **Authoritative sources over invention.** CLDR RBNF spellout rules + (number words, vigesimal, CJK scales) and dateFields (relative-date vocab) + are the canonical data sources — Unicode-licensed, embeddable. Duckling's + per-language `Corpus.hs` gold sets (BSD-3) validate our corpus rows. + Recognizers-Text YAML documents hazard patterns (CJK ambiguity filters). +4. **Two-way and ambiguity rules unchanged.** New idioms are ADDITIVE corpus + entries. Hazards (三/多 polysemy, 斤 regional values, "half seven" + UK-vs-de semantics) follow the D4 honest-ambiguity policy. + +## Wave 1 (this pass) + +Engine unlocks, each consumed through new optional pack fields: + +- **Romance number composition** — tens + and-word + ones (`treinta y + cinco`); bare scale words (`cien gramos`, `mil metros`); single-word + hundreds via `numberWords.composed` (`quinientos`, `quatre-vingt-dix` + exhaustive vigesimal per CLDR RBNF); spoken decimal separators + (`dos coma cinco`) via `numberWords.decimalWords`. +- **Multi-word approximants** — leading `grammar.approximatePhrases` + (`más o menos`, `à peu près`, `por volta de`); trailing data fixes ride the + existing `trailingApproxPhrases` (`y pico`, `e pouco`). +- **CJK segmentation** — intra-token number sub-parser (三十五, 一百五十万, + elliptical 一百五 = 150, mixed 3万5千), unit/particle splitting + (三公斤, 5公斤左右), wave-dash ranges (5〜10), adjacent-number ranges + (三四个), post-unit 半. +- **Localized date grammar** — pack-owned spoken clock vocabulary + (`las tres menos cuarto`, `deux heures et quart`, 午後3時半, `quarter of + five`), period-edge phrases (`a finales de mes`, `fin juillet`, 月底), + weekday-offset phrases (`lundi en huit`, `Monday week`), after-next/ + before-last modifiers (再来週/先々週), day+day-part compounds + (`tomorrow morning`), duration parsing honoring pack unit words + (`2 horas`, `hora y media`). +- **Deepened packs** for es/fr/pt/zh/ja/en-GB covering every data-only gap + idiom from the research, plus per-locale corpus gates. + +Deliberately deferred to wave 2 (backlog): CLDR-generated unit-alias packs at +scale, new locales (de/it/nl/ko/ar incl. RTL + Eastern Arabic numerals), +localized humanize output (D66 deferral stands), counters/classifiers beyond +filler handling, height/weight elliptical speech ("uno ochenta"), currency +slang ("a grand", "5 lucas"). + +## Acceptance + +- Per-locale corpus gates run in `bun run check`; every wave-1 idiom class has + rows; existing English corpus interpretation unchanged (ADDITIVE only). +- All very-common idioms from the research parse in their language, with + round-trip where the two-way guarantee applies. +- Size: locale-pack standalone/marginal budgets and engine entries + recalibrate once, with a D-entry recording measured costs (D62/D64/D66 + pattern) — never silently. diff --git a/plans/backlog.md b/plans/backlog.md index 722c76b..c26be5b 100644 --- a/plans/backlog.md +++ b/plans/backlog.md @@ -238,3 +238,63 @@ recorded here as a parking-lot idea, not a commitment. See Schema and throw if any object has `additionalProperties:true` or missing `required`; catches the `passthrough:true` footgun at definition time for OpenAI/Anthropic strict mode. (ai-structured-output.md addendum, 2026-07-09) + +## From locale idiom research pass 2026-07-09 (wave 2, deferred) + +Items deferred from plan 033 wave 1. See `wiki/research/locale-idioms.md` for +full context. + +- **New locales de/it/nl/ko/ar + RTL/Eastern-Arabic numerals** — the five + highest-demand missing locales after the shipped six. Arabic requires RTL text + handling in spans and Eastern-Arabic digit normalization (٠-٩ -> 0-9). German + needs 'halb sieben' = 6:30 (half-before semantics). Korean shares CJK scale + structure. (competitive-landscape.md: chrono has 14 locales vs lingo's 7) +- **CLDR-generated unit-alias packs at scale** — use `cldr-units-full` JSON to + auto-generate unitAliases entries (singular + plural + gender variants) for all + locale packs instead of manual curation. Requires a build-time script and a + decision on which CLDR unit subset to include per pack (all ~300 units is too + large for size budgets). +- **Localized humanize/format output** — `humanizeDate` and `format` currently + emit English text regardless of locale. Producing locale-appropriate output + ("hace 3 dias", "il y a 3 jours") requires either CLDR pattern templates or + per-locale format strings in the date vocab pack. Deferred by D66; the two-way + guarantee means locale humanize output must also re-parse in that locale. +- **Height/weight elliptical speech** — 'uno ochenta' (es, 1.80m), 'one eighty' + (en, 180), 'un metre quatre-vingts' (fr, 1.80m). Implied-unit compound numbers + where the speaker drops the unit and uses a decimal-shorthand convention. Deeply + ambiguous without domain context (height vs generic number); needs a + context-hint or dedicated compound-height mode. +- **Currency slang multipliers** — 'a grand' / 'five grand' (en, x1000), '5 + lucas' (es-CL, x1000 CLP), 'un baton' (fr slang, x10000). These are + kind-scoped scale words that only apply to currency. The current + numberWords.scales is kind-agnostic; needs either kind-scoped scales or + dedicated currency-multiplier entries. +- **Counters/classifiers as first-class concept** — Japanese 個/人/本/匹 and + Chinese 个/只/条 are mandatory between numbers and nouns. Currently treated as + unknown tokens or fillers. Elevating them to a `classifierWords` pack field + consumed by the quantity parser would let CJK expressions with counters parse + cleanly (三個, 五人, 二匹). +- **`excludePatterns` / ambiguity-filter pack field** — per-locale list of + strings where numeral characters appear in non-numeric words (Chinese: 十足, + 大陆, 放肆; Japanese: 一応, 二度手間). Prevents false-positive number matches + in free text. Pattern from Microsoft/Recognizers-Text AmbiguityFiltersDict. +- **Script to mine duckling Corpus.hs into corpus rows** — a build-time tool + that parses Duckling's Haskell corpus files and emits TypeScript + `tests/corpus/locale--source.mjs` arrays. Priority: ZH/JA Numeral, FR + Time, EN Quantity. BSD-3 requires copyright notice in generated files. + +## Wave-1 idiom deferrals (2026-07-09, D69) + +- **No-space CJK trailing approx** — `五公斤左右` / `五キロぐらい` still fail: + the suffix particle merges into the unit token; the unit matcher needs + approx-suffix stripping for CJK word tokens (spaced forms work today). +- **CJK spoken clock** — `午後3時半` / `三点一刻` need the D69 clock fields to + compose with the CJK tokenizer path; deferred from the date-grammar package. +- **Unit-first implied-1 quantities** — `kilo y medio`, `hora y media`, + `metro e meio`: unit-before-number with implied quantity 1; needs a + quantity-grammar seam, was cut from D69 for size. +- **CJK decimal-word composition** — `三点五公斤` (3.5 kg): the 点 decimal word + needs to work inside the CJK number walker, not just as a spaced token. +- **`个` classifier before duration units in ranges** — `三四个小时` composes + adjacent-number range + classifier + unit; blocked on the classifier field + (see counters/classifiers item above). diff --git a/wiki/decisions.md b/wiki/decisions.md index d3d38b6..1e2db62 100644 --- a/wiki/decisions.md +++ b/wiki/decisions.md @@ -486,3 +486,61 @@ standalone recalibrates 38.2 → 38.3 kB (measured 38.26 after golfing) for the humanize + zone correctness weight — correctness is product, not bloat (D14 pattern). The corpus gained only additive entries; the property suite keeps one documented exclusion (bare `C` charge-vs-Celsius, D43). + +**D68 · 2026-07-09 · Wave-1 idiom engine: new pack-data seams, no language +branches.** The locale idiom research pass (335 gap idioms across es/fr/pt/zh/ja/en, +`wiki/research/locale-idioms.md`, plan 033) showed whole idiom classes blocked by +engine assumptions no pack data could reach. The fix follows D5 strictly — new +OPTIONAL pack fields consumed by shared grammar, zero per-language parser code: +`numberWords.bareScales` (bare `cien`/`mil`/`cem` open a number; English +`hundred` deliberately still requires `a`), `numberWords.composed` +(longest-match exact compounds: `quinientos`, exhaustive French vigesimal per +CLDR RBNF), `numberWords.decimalWords` (`dos coma cinco`, `trois virgule +quatorze`; English gains `point` via the same table), tens+andWords+ones +composition (`treinta y cinco` — guarded so `entre cinco y diez` stays a range), +and `grammar.approximatePhrases` (`más o menos`, `à peu près`). CJK gets a +profile-table-driven number walker (`number/cjk.ts`): sub-token segmentation +(三公斤 → 3 kg, the tokenizer's merged-CJK-token blocker), 万/亿 group algebra +(三百五十万 = 3.5M), elliptical shorthands (一百五 = 150, 三万五 = 35 000), +mixed digit+scale (3万5千), wave-dash range folding (5〜10), adjacent-number +implicit ranges (七八天 = 7–8 days; 三三 stays rejected), and post-unit 半 +(两公斤半 = 2.5 kg). D47's glued-`1M` rejection holds — digit+scale composition +fires only on CJK scale characters. Chosen over chrono-node's imperative +per-locale parser classes (14 locales and still no vigesimal or `en huit` — +imperative locale code demonstrably doesn't scale idiom coverage). Cost after a +golf pass: `full` 36.9 → 38.3 kB (measured 38.19), `./core` 24.2 → 25.6 +(measured 25.46), `./date` standalone 38.3 → 39.7 (measured 39.51) — the +engine mechanisms live in the shared number/parse layer so BYO-registry cores +get them; capability is product (D14). Pack data itself stays tree-shakeable in +per-locale entries (their budgets unchanged). Corpus: zero changes to existing +entries. + +**D69 · 2026-07-09 · Wave-1 idiom data + localized date grammar + per-locale +corpus gates.** Completing plan 033's wave 1 on top of D68: (1) the date module +gains pack-owned spoken-clock vocabulary (`clockPastWords`/`clockToWords`/ +`clockMinuteWords` — `las tres menos cuarto`, `quarter of five`, `half seven`), +`periodEdgePhrases` (start=day 1, mid=15, end=last day: `fin juillet`, 月底, +`end of the month`), `weekdayOffsetPhrases` (`lundi en huit`, `Monday week`, +`Tuesday fortnight`), `afterNext`/`beforeLast` modifier slots (再来週, 先々週, +`the week after next`), `dayPartWords` compounds (`tomorrow morning`), and +duration parsing that honors pack unit words (`2 horas`) — English defaults +preserved behavior-identically. (2) The es/fr/pt packs fill the D68 fields +(exhaustive French vigesimal from CLDR RBNF, ES/PT compound hundreds, `coma`/ +`virgule`/`virgula` decimals, `y pico`/`e pouco` trailing approx, `más o menos` +class phrases, `pasado mañana`/`avant-hier`/`anteontem` offsets) and zh/ja fill +the CJK walker tables (两/幺/点/半, duration counters 个小时/時間, trailing +左右/上下, 前天/一昨日/大後天 offsets, 月底/年初 edges, 円→JPY). Traditional +units 斤/两/里/坪 are deliberately NOT registered: locale aliases map words to +EXISTING unit ids only, 斤=500g needs a real unit definition and 两 collides +with the numeral 2 (D4 honesty; backlog). (3) Locale behavior is now +corpus-gated like English: `tests/corpus/locale--source.mjs` → +`locale--contract-v1.json` per pack (184 rows: es 42, fr 37, pt 42, zh 30, +ja 34 at landing), discovered generically by `corpus-diff.mjs` and enforced in +`bun run check`. Budgets recalibrate once for the wave: packs es 1.8/fr 1.9/ +pt 1.7/zh 1.3/ja 1.4 kB standalone (romance marginal 3.9, cjk 2.0, all 5.8), +`./date` standalone 39.7 → 41.0 (measured 40.85), `./date` marginal 13.1 → +14.4, `./ai` 16.1 → 17.4 (the /ai entry bundles the date engine). Main-entry +and `./core` D68 budgets hold. Deferred to wave 2 (backlog): no-space CJK +trailing approx (五公斤左右 needs suffix-stripping in the unit matcher), CJK +spoken clock (午後3時半), unit-first implied-1 (`hora y media`, `kilo y +medio`), 三点五 CJK decimal composition, `uno ochenta` elliptical heights. diff --git a/wiki/inspiration.md b/wiki/inspiration.md index 6d0e035..98a9583 100644 --- a/wiki/inspiration.md +++ b/wiki/inspiration.md @@ -156,3 +156,12 @@ license obligation; we do not copy code without noting it here explicitly. | [Base UI](https://base-ui.com) Combobox (v1.6.0) | MIT | The ARIA combobox contract (`role="combobox"` + `aria-autocomplete="list"` + `aria-activedescendant` pointing at highlighted option) as the accessibility pattern for lingo's DOM completions popup — keeps DOM focus on the input while screen readers navigate the list. Applied to the planned completions ARIA wiring in `packages/lingo/src/dom/controller.ts`. | | [FFmpeg FATE](https://fate.ffmpeg.org) + plan 010 | LGPL-2.1 (the practice pattern, not code) | Seeded property round-trip testing: encode-then-decode oracle (FATE's `enc_dec_pcm` pattern) as the direct model for lingo's two-way guarantee property test — generate random magnitudes, `format()`, `parse()`, assert `base` equality within epsilon. Validates the plan 010 layer-2 spec from independent prior art. | | [es-toolkit](https://es-toolkit.slash.page) | MIT | Per-entry size/benchmark publishing as a documentation practice: each function measured individually with esbuild min+gzip, results surfaced on the site with head-to-head comparisons. Lingo already has the measurement infrastructure (`scripts/size.mjs`, `scripts/bench.mjs`); the idea is to pipe those numbers into the published docs site as a future docs-site item. | + +## Locale idiom coverage (research pass 2026-07-09) + +| source | license/type | what we took | +|--------|--------------|--------------| +| [CLDR RBNF](https://github.com/unicode-org/cldr/tree/main/common/rbnf) (Rule-Based Number Format XML) | Unicode License (ICU), MIT-compatible | Canonical number-word spellout rules for 100+ locales as the authoritative source for generating exhaustive `numberWords.ones`/`tens`/`scales` entries per locale pack. Specifically: French vigesimal composition rules (soixante-dix through quatre-vingt-dix-neuf), CJK scale multiplication patterns, Spanish gender-inflected compound hundreds (doscientos/doscientas). Adopted as the single source of truth for number-word table generation rather than manual curation. | +| [CLDR dateFields JSON](https://github.com/unicode-org/cldr-json/tree/main/cldr-json/cldr-dates-full) | Unicode License, MIT-compatible | Structured per-locale relative-date vocabulary (dayOffset -2 to +2, relative time pattern templates, calendar period labels) directly mappable to lingo's `dayOffsets`, `calendarPeriodPhrases`, and `relative` pack fields. Fills missing entries like avant-hier (fr), pasado manana (es), 一昨日/明後日 (ja), 前天/后天 (zh) from a machine-parseable canonical source instead of manual inventory. | +| [facebook/duckling Corpus.hs](https://github.com/facebook/duckling/tree/main/Duckling) per-language corpus files | BSD-3-Clause (permissive, MIT-compatible for derived test data with copyright notice) | Gold-standard (input_text, expected_value) test pairs across 49 languages x 15 dimensions as validation sets for lingo's per-locale corpus gates. Priority targets: ZH/JA Numeral (validates scale composition), FR Time (validates vigesimal + spoken clock), EN/ZH Quantity. A script will mine Corpus.hs into TypeScript test-row format. Not embedding Duckling code — only the test expectations as validation oracles. | +| [Microsoft/Recognizers-Text](https://github.com/microsoft/Recognizers-Text) AmbiguityFiltersDict pattern | MIT | The false-positive exclusion mechanism (a per-locale dictionary of strings where numeral characters appear in non-numeric words: '十足', '大陆', '放肆' in Chinese) as the model for a planned `excludePatterns` pack field. Prevents CJK numerals from firing on idiomatic compounds that happen to contain numeral kanji. Adopted as a design pattern, not as embedded data. | diff --git a/wiki/research/locale-idioms.md b/wiki/research/locale-idioms.md new file mode 100644 index 0000000..b02f577 --- /dev/null +++ b/wiki/research/locale-idioms.md @@ -0,0 +1,950 @@ +# Locale idiom inventory + +Research pass 2026-07-09 (multi-agent, source-level). Six native-level language +studies (es, fr, pt, zh, ja, en) producing ~335 test-precise idioms, a prior-art +source audit, an engine-gap analysis with file:line evidence, and a field-coverage +matrix. Agent-researched: idioms are agent-curated from native corpora, not +independently verified by native speakers. + +Companion plan: `plans/033-locale-idiom-coverage.md`. + +--- + +## Spanish (es) — 58 idioms + +Register landscape: rich spoken-number system with mandatory connectors +("treinta y cinco"), bare scales ("cien", "quinientos"), fused compounds +(veinti- forms for 21-29). Multi-word approximate phrases dominate informal +speech. Time expressions use article + cardinal + additive/subtractive fractions +with period-of-day suffixes. + +### number-word + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| veintiuno | value: 21 | very-common | numberWords.ones | Fused form. Variants veintidos-veintinueve all missing from ones. | +| veintiun kilos | 21 kg | very-common | numberWords.ones | Apocopated before masculine nouns. | +| doscientos gramos | 200 g | very-common | ENGINE | Bare compound-hundred; consumedAny gate blocks it. Feminine doscientas also needed. | +| quinientos gramos | 500 g | very-common | ENGINE | Same bare-scale blocker; needs doscientos-novecientos as ones(200-900) or engine fix. | +| cien gramos | 100 g | very-common | ENGINE | Bare scale without preceding number; consumedAny=true gate. | +| mil quinientos metros | 1500 m | common | ENGINE | Depends on fixing quinientos + bare mil. | +| dos coma cinco kilos | 2.5 kg | common | ENGINE | 'coma' as spoken decimal separator; no decimalWords field exists. | + +### compound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| treinta y cinco kilos | 35 kg | very-common | ENGINE | Parser checks only hyphen+ones after tens, never andWords+ones. Affects all 31-99. | +| uno ochenta | 1.80 m (height) | common | ENGINE | Number-word parser sums 1+80=81; needs domain-specific decimal-shorthand heuristic. | + +### fraction + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| kilo y medio | 1.5 kg (implied 1) | very-common | ENGINE | Unit-first with implied quantity 1. | +| medio kilo | 0.5 kg | very-common | numberWords.fractionWords | Already works. | +| un cuarto de kilo | 0.25 kg | very-common | numberWords.fractionWords | Already works. | +| dos kilos y medio | 2.5 kg | very-common | numberWords.fractionWords | Already works. | +| 3 horas y media | 3.5 hours | very-common | numberWords.fractionWords | Should work via compound 'and a half' tail. | +| media docena | 6 (half a dozen) | common | ENGINE | No article between fraction and dozen word in Spanish. | + +### duration + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| hora y media | 1.5 h | very-common | ENGINE | Unit-first implied 1 + parseDuration locale gap. | +| media hora | 0.5 h = 30 min | very-common | ENGINE | parseDuration does not load locale unit aliases. | +| un cuarto de hora | 15 min | very-common | ENGINE | Same parseDuration locale gap. | +| dos horas | 2 h | very-common | ENGINE | parseDuration doesn't consult locale pack 'horas' alias. | +| la quincena | 15 days (fortnight) | common | ENGINE | No custom-duration-unit mechanism. | + +### approx-qualifier + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| mas o menos 5 kilos | ~5 kg | very-common | ENGINE | 3-word phrase; approximateWords single-word only. | +| unos 5 kilos | ~5 kg | very-common | grammar.approximateWords | Data fix: add 'unos'/'unas'. | +| como unos 5 kilos | ~5 kg | common | grammar.approximateWords | 'como' conflicts with conversionWords. | +| cosa de 5 kilos | ~5 kg | occasional | ENGINE | Multi-word approximate phrase. | +| por ahi de 5 kilos | ~5 kg | common | ENGINE | Multi-word approximate phrase. | +| alrededor de 5 kilos | ~5 kg | very-common | grammar.qualifierSkipAfterApprox | 'de' breaks chain after approx word; add 'de' to skip list. | +| cerca de 5 kilos | ~5 kg | common | grammar.qualifierSkipAfterApprox | Same 'de' skip fix. | +| 5 kilos y pico | ~5+ kg | very-common | grammar.trailingApproxPhrases | Data fix: add 'y pico'. | +| cinco kilos y pico | ~5+ kg | very-common | grammar.trailingApproxPhrases | Same. | +| 5 kilos y tantos | ~5+ kg | common | grammar.trailingApproxPhrases | Data fix: add 'y tantos'. | +| 5 y algo kilos | ~5 kg | common | grammar.trailingApproxPhrases | Data fix: add 'y algo'. | + +### fuzzy-amount + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| veintipico kilos | ~20+ kg | common | ENGINE | Fused approx compound; value is a range, not scalar. | +| un par de kilos | ~2 kg | very-common | numberWords.fuzzyAmounts | Data fix: add 'par' {value:2, spread:[2,3]}. | +| unos cuantos kilos | ~4-5 kg | common | ENGINE | Multi-word fuzzy. If 'unos' is approxWord, 'cuantos' as fuzzyAmount might parse. | +| cuarenta y tantos anos | ~40-49 | common | ENGINE | tens + fuzzy suffix; cannot express as simple data. | + +### range-bound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 5 o 6 kilos | range 5-6 kg | very-common | grammar.rangeAlternativeWords | Data fix: add 'o'. | +| de 5 a 10 kilos | range 5-10 kg | very-common | grammar.rangeFromWords | Data fix: add 'de'. | +| entre 5 y 10 kilos | range 5-10 kg | very-common | grammar | Already works. | +| desde 5 hasta 10 kilos | range 5-10 kg | common | grammar | Already works. | +| al menos 5 kilos | at least 5 kg | very-common | grammar.boundPhrases | Already works. | + +### number-word (decimal/format) + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 2,5 kilos | 2.5 kg | very-common | defaults.numberFormat | Already works (comma-decimal policy). | +| pesa ochenta kilos | 80 kg | common | grammar.globalFillers | Data fix: add 'pesa'/'mide' to fillers. | + +### date-relative + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| pasado manana | dayOffset +2 | very-common | date.dayOffsets | Data fix. | +| anteayer | dayOffset -2 | very-common | date.dayOffsets | Data fix. | +| antier | dayOffset -2 | common | date.dayOffsets | Regional (Mexico/Central America). | +| el lunes que viene | next Monday | very-common | date.modifiers | Already works. | +| dentro de quince dias | in 15 days | very-common | date.relative | Already works. | +| ahorita | right now / very soon | very-common | date.dayTimePhrases | Regional (Mexico). Data fix. | +| el otro dia | ~3 days ago | very-common | date.dayOffsets | Could map dayOffset -3. | +| hace un rato | ~30-60 min ago | very-common | ENGINE | Informal duration unit with no precise mapping. | +| a mediados de julio | mid-July | common | ENGINE | English-only regex for period-edge phrases. | +| a finales de mes | end of month | common | ENGINE | Same hardcoded English regex. | +| a principios de mes | beginning of month | common | ENGINE | Same. | +| el fin de semana | this weekend | very-common | ENGINE | Hardcoded English 'weekend' match. | +| de madrugada | early morning (~4h) | common | date.dayTimePhrases | Data fix. | + +### time-of-day + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| las dos y media | 2:30 | very-common | ENGINE | Romance clock grammar not supported. | +| las tres menos cuarto | 2:45 | very-common | ENGINE | Subtractive time (menos=minus). | +| a las siete de la tarde | 19:00 | very-common | ENGINE | Period-of-day suffixes not recognized. | + +--- + +## French (fr) — 60 idioms + +Register landscape: vigesimal number system (60+N for 70-79, 4x20 for 80, +4x20+N for 90-99) is the dominant structural gap. Belgian/Swiss forms +(septante/huitante/nonante) already work. Multi-word approximate phrases +universal in speech. Time uses "N heures M" pattern with additive/subtractive +fractions. + +### number-word (vigesimal) + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| quatre-vingts kilos | 80 kg | very-common | ENGINE | 4*20 multiplicative; parser treats tens as additive only. | +| quatre-vingt-dix kilos | 90 kg | very-common | ENGINE | 4*20+10 cascaded composition. | +| soixante-dix kilos | 70 kg | very-common | ENGINE | 60+10; 'dix'=10 rejected by tens-hyphen-ones branch (requires <10). | +| soixante-quinze kilos | 75 kg | very-common | ENGINE | 60+15; same branch rejection. | +| soixante et onze kilos | 71 kg | very-common | ENGINE | 60+11 with 'et' liaison; wrong path. | +| quatre-vingt-onze kilos | 91 kg | very-common | ENGINE | 4*20+11. | +| vingt et un kilos | 21 kg | very-common | ENGINE | 'et' after tens expects scale, not ones. | +| cent kilos | 100 kg | very-common | numberWords.scales | Bare scale; consumedAny gate blocks. | +| mille kilos | 1000 kg | very-common | numberWords.scales | Same bare-scale issue. | + +### number-word (decimal) + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| deux virgule cinq kilos | 2.5 kg | very-common | ENGINE | 'virgule' as decimal-point word; no decimalWords field. | + +### regional-variant + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| septante-cinq kilos | 75 kg (BE/CH) | very-common | numberWords.tens | Already works. | +| nonante-deux kilos | 92 kg (BE/CH) | very-common | numberWords.tens | Already works. | +| huitante kilos | 80 kg (CH) | common | numberWords.tens | Already works. | + +### fuzzy-amount + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| une dizaine de kilos | ~10 kg | very-common | numberWords.fuzzyAmounts | Data fix: dizaine {value:10, spread:[8,12]}. | +| une vingtaine de kilos | ~20 kg | very-common | numberWords.fuzzyAmounts | vingtaine {value:20, spread:[18,25]}. | +| une centaine de kilos | ~100 kg | common | numberWords.fuzzyAmounts | centaine {value:100, spread:[80,120]}. | +| une trentaine de kilos | ~30 kg | common | numberWords.fuzzyAmounts | trentaine {value:30, spread:[25,35]}. | +| un millier de kilos | ~1000 kg | common | numberWords.fuzzyAmounts | millier {value:1000, spread:[800,1200]}. | +| une cinquantaine de kilometres | ~50 km | common | numberWords.fuzzyAmounts | cinquantaine {value:50, spread:[45,55]}. | + +### approx-qualifier + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| a peu pres 5 kilos | ~5 kg | very-common | ENGINE | Multi-word; no approximatePhrases field. | +| dans les 5 kilos | ~5 kg | very-common | ENGINE | 'dans' conflicts with date futurePrefixes. | +| grosso modo 5 kilos | ~5 kg | common | ENGINE | Two-word phrase. | +| plus ou moins 5 kilos | ~5 kg | common | ENGINE | Three-word; 'plus'/'moins' have bound semantics. | +| pas loin de 5 kilos | ~5 kg | common | ENGINE | Multi-word approximate expression. | +| aux alentours de 5 kilos | ~5 kg | common | ENGINE | Four-word phrase. | +| 5 kilos environ | ~5 kg | very-common | grammar.trailingApproxWords | Data fix: add 'environ'. | +| 5 kilos a peu pres | ~5 kg | common | grammar.trailingApproxPhrases | Data fix: add 'a peu pres'. | +| un peu plus de 5 kilos | slightly >5 kg | very-common | grammar.qualifierSoftenerPhrases | Partial engine gap: softener+bound composition. | +| un peu moins de 5 kilos | slightly <5 kg | very-common | grammar.qualifierSoftenerPhrases | Same. | + +### range-bound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 5 ou 6 kilos | range 5-6 | very-common | grammar.rangeAlternativeWords | Data fix: add 'ou'. | + +### compound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| un metre quatre-vingts | 1.80 m | very-common | ENGINE | Compound + vigesimal subunit. | +| un metre quatre-vingt-deux | 1.82 m | very-common | ENGINE | Same vigesimal blocker in subunit. | + +### fraction + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| la moitie d'un kilo | 0.5 kg | common | ENGINE | 'la moitie de' not recognized as fraction lead. | + +### duration + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| une heure et demie | 1.5 h | very-common | ENGINE | As duration works; as clock time (1:30) fails. | +| un quart d'heure | 15 min | very-common | ENGINE | Apostrophe tokenization + duration module gap. | +| une demi-heure | 30 min | very-common | ENGINE | Hyphenated fraction-unit pattern. | + +### date-relative + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| apres-demain | dayOffset +2 | very-common | date.dayOffsets | Data fix. | +| avant-hier | dayOffset -2 | very-common | date.dayOffsets | Data fix. | +| ce soir | this evening ~21h | very-common | date.dayTimePhrases | Data fix. | +| hier soir | yesterday evening | very-common | date.dayTimePhrases | Data fix. | +| demain soir | tomorrow evening | very-common | date.dayTimePhrases | Data fix. | +| cet apres-midi | this afternoon ~15h | very-common | date.dayTimePhrases | Data fix. | +| le week-end prochain | next weekend | very-common | date.periodWords | Data fix: add 'week-end'/'weekend' to week. | +| dans quinze jours | in 15 days | very-common | date.relative | Already works. | +| la semaine prochaine | next week | very-common | date.periodWords + modifiers | Already works. | +| lundi en huit | Monday next week | common | ENGINE | Weekday + 'en huit' offset; no pack field. | +| mardi en quinze | Tuesday in 2 weeks | occasional | ENGINE | Same. | +| d'ici lundi | by Monday | common | ENGINE | No deadline-by mechanism. | + +### date-absolute + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| debut juillet | early July | common | ENGINE | English-only sub-month regex. | +| mi-juillet | mid-July | common | ENGINE | Regex only matches literal 'mid'. | +| fin juillet | end of July | common | ENGINE | Same. | +| le 1er juillet | July 1st | very-common | ENGINE | French ordinal suffix 'er' not stripped. | +| le premier juillet | July 1st | common | ENGINE | Ordinal word for day not supported. | + +### time-of-day + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| quatorze heures trente | 14:30 | very-common | ENGINE | Word-based clock time not supported. | +| deux heures et quart | 2:15 | very-common | ENGINE | Fraction-based clock. | +| trois heures moins le quart | 2:45 | very-common | ENGINE | Subtractive 'moins'. | +| midi et demi | 12:30 | very-common | ENGINE | Alias + fraction tail not composable. | +| vers 15h | ~15:00 | very-common | ENGINE | Approximate time prefix not handled. | + +### colloquial + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| je fais du 42 | size 42 | common | ENGINE | Clothing size idiom; outside grammar. | +| en fin d'apres-midi | ~17-18h | common | date.dayTimePhrases | Data fix (needs apostrophe normalization check). | + +--- + +## Portuguese (pt) — 58 idioms + +Register landscape: mandatory "e" connector between tens and ones (like Spanish +"y"), bare scales (cem/mil), compound hundreds (duzentos-novecentos). Brazilian +vs European Portuguese variants for teens (dezesseis/dezasseis). Spoken clock uses +"X para as Y" (X to Y) and "N e meia" (N:30). + +### number-word + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| vinte e cinco quilos | 25 kg | very-common | ENGINE | Parser only checks hyphen after tens, never andWords. | +| cento e cinquenta gramas | 150 g | very-common | ENGINE | 'e' captured as rangeAndWord before number-word parser sees it. | +| cinquenta e tres quilos | 53 kg | very-common | ENGINE | Same tens+e+ones failure. | +| cem quilos | 100 kg | very-common | ENGINE | Bare scale blocked by consumedAny. | +| mil quilos | 1000 kg | very-common | ENGINE | Same. | +| mil e quinhentos metros | 1500 m | common | numberWords.ones | Needs engine fix + 'quinhentos' in pack. | +| duzentos gramas | 200 g | very-common | numberWords.ones | Missing compound hundred. | +| trezentos metros | 300 m | common | numberWords.ones | Missing. | +| quatrocentos quilos | 400 kg | common | numberWords.ones | Missing. | +| quinhentos gramas | 500 g | very-common | numberWords.ones | Missing. | +| seiscentos metros | 600 m | common | numberWords.ones | Missing. | +| setecentos metros | 700 m | common | numberWords.ones | Missing. | +| oitocentos metros | 800 m | common | numberWords.ones | Missing. | +| novecentos metros | 900 m | common | numberWords.ones | Missing. | +| dois virgula cinco quilos | 2.5 kg | common | ENGINE | 'virgula' as decimal word; no decimalWords field. | + +### regional-variant + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| dezessete quilos | 17 kg (BR) | very-common | numberWords.ones | Pack has 'dezassete' (PT) not 'dezessete' (BR). | +| dezasseis quilos | 16 kg (PT) | very-common | numberWords.ones | Pack has 'dezesseis' (BR) not 'dezasseis' (PT). | +| dezanove quilos | 19 kg (PT) | very-common | numberWords.ones | Pack has 'dezenove' (BR) not 'dezanove' (PT). | + +### fraction + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| dois quilos e meio | 2.5 kg | very-common | grammar.compoundJoinWords | Already works. | +| uma hora e meia | 1.5 h | very-common | grammar.compoundJoinWords | Already works. | +| meia duzia de ovos | 6 (half dozen) | common | ENGINE | No article between fraction and dozen word. | +| um quarto de hora | 15 min | common | ENGINE | parseDuration lacks locale support. | + +### approx-qualifier + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| uns 5 quilos | ~5 kg | very-common | grammar.approximateWords | Data fix: add 'uns'/'umas'. | +| umas 3 caixas | ~3 boxes | very-common | grammar.approximateWords | Feminine form. | +| mais ou menos 5 quilos | ~5 kg | very-common | ENGINE | Multi-word phrase; no field. | +| por volta de 3 quilos | ~3 kg | common | ENGINE | Multi-word phrase. | +| cerca de 5 quilos | ~5 kg | very-common | grammar.qualifierSkipAfterApprox | Add 'de' to skip list. | +| vinte e poucos quilos | ~20-29 kg | very-common | ENGINE | Trailing 'e pouco' blocked by rangeAndWord 'e'. | +| vinte e tantos quilos | ~20-29 kg | common | ENGINE | Same 'e' conflict. | + +### range-bound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 5 ou 6 quilos | range 5-6 | common | grammar.rangeAlternativeWords | Data fix: add 'ou'. | + +### fuzzy-amount + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| um par de quilos | ~2 kg | common | numberWords.fuzzyAmounts | 'par' with articles+ofWords path. | +| bocado | a bit (PT-PT) | common | numberWords.fuzzyAmounts | Context-dependent. | + +### compound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| um e oitenta | 1.80 m (height) | very-common | ENGINE | Implied meter + centimeter subunit. | +| um metro e oitenta | 1.80 m | very-common | grammar.compoundJoinWords | Add 'e' to compoundJoinWords. | + +### duration + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| hora e meia | 1.5 h | very-common | ENGINE | parseDuration locale gap. | +| meia hora | 30 min | very-common | ENGINE | Same. | +| quinzena | 15 days | common | date.unitWords | No custom-duration-unit mechanism. | + +### date-relative + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| anteontem | dayOffset -2 | very-common | date.dayOffsets | Data fix. | +| depois de amanha | dayOffset +2 | very-common | date.dayOffsets | Data fix (multi-word key). | +| fim de semana | weekend | very-common | ENGINE | Hardcoded English 'weekend'. | +| em quinze dias | in 15 days | very-common | date.relative | Already works. | +| segunda que vem | next Monday | very-common | date.modifiers | Already works. | +| daqui a tres dias | in 3 days | very-common | ENGINE | parseDate doesn't resolve locale number words in offset. | +| semana retrasada | week before last | occasional | ENGINE | No 'before-last' modifier. | +| inicio de julho | early July | common | ENGINE | English-only edge-period regex. | +| fim de julho | late July | common | ENGINE | Same. | +| meio de julho | mid-July | common | ENGINE | Same. | +| no comeco do mes | beginning of month | common | ENGINE | Same. | + +### time-of-day + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| duas e meia | 2:30 | very-common | ENGINE | PT spoken clock not supported. | +| tres e quinze | 3:15 | common | ENGINE | Same. | +| quinze para as tres | 2:45 | very-common | ENGINE | 'para' not in subtraction prepositions. | +| meio-dia e meia | 12:30 | very-common | ENGINE | Alias + fraction not composable. | +| as sete da noite | 19:00 | very-common | ENGINE | Period-of-day suffixes not supported. | +| de madrugada | early morning | common | date.dayTimePhrases | Data fix. | +| la pelas tres | ~15:00 | common | ENGINE | Approximate time marker not handled. | +| as duas da tarde | 14:00 (PT-PT) | very-common | ENGINE | Same period-of-day gap. | +| dez para as oito | 7:50 | common | ENGINE | Same. | +| depois de amanha de manha | day+2 morning | common | date.dayTimePhrases | Compound dayOffset + time phrase. | + +--- + +## Chinese, Simplified Mandarin (zh) — 55 idioms + +Register landscape: fundamentally different numeral composition from Western +languages. Numbers built by multiplying and adding CJK scale characters +(百/千/万/亿). No spaces between tokens. Classifiers (个/只/条) mandatory between +numbers and nouns. Elliptical shorthand pervasive (一百五 = 150). Traditional +units (斤/两/里/亩) still in daily use. + +### number-word + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 三公斤 | 3 kg | very-common | ENGINE | Tokenizer merges into one token; number parser cannot split. | +| 十五公斤 | 15 kg | very-common | ENGINE | 十五 positional CJK; token splitting needed. | +| 一百五十公斤 | 150 kg | very-common | ENGINE | Multi-scale composition in one token. | +| 一百五 | 150 (elliptical) | very-common | ENGINE | X百Y = X*100 + Y*10 shorthand. | +| 三万五 | 35000 (elliptical) | very-common | ENGINE | X万Y = X*10000 + Y*1000. | +| 两 | 2 (colloquial before units) | very-common | ENGINE | In ones already but CJK merge blocks matching. | +| 三点五公斤 | 3.5 kg | common | ENGINE | 点 as CJK decimal point; no mechanism. | + +### compound (mixed Arabic + CJK) + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 3万5 | 35000 | very-common | ENGINE | 万 as post-digit scale not recognized. | +| 3万5千 | 35000 | common | ENGINE | Mixed digits + CJK scales. | + +### numeral-system + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| ３５公斤 | 35 kg (full-width) | common | normalizer | Already works (NFKC normalization). | + +### fraction + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 两公斤半 | 2.5 kg | very-common | ENGINE | Post-unit 半 pattern; no mechanism for suffix fraction. | +| 一个半小时 | 1.5 h | very-common | ENGINE | N+个+半+unit pattern. | +| 半个月 | 0.5 months | very-common | ENGINE | Leading 半 with classifier; one merged token. | +| 两斤半 | 1.25 kg (2.5 jin) | very-common | ENGINE | Traditional unit + post-unit half. | + +### approx-qualifier + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 十来个 | ~10 | very-common | ENGINE | 来 as intra-token approximate suffix. | +| 二十多公斤 | 20+ kg | very-common | ENGINE | 多 as post-number 'more than' marker; intra-token. | +| 一百多公斤 | 100+ kg | common | ENGINE | Same 多 pattern at higher scale. | +| 五公斤左右 | ~5 kg | very-common | grammar.approximateWords | Works with Arabic digits; fails with CJK numerals. | +| 大概五公斤 | ~5 kg | very-common | grammar.approximateWords | Works with Arabic digits. | +| 差不多五公斤 | ~5 kg | very-common | grammar.approximateWords | Missing from current pack. Multi-char no-space. | +| 五公斤上下 | ~5 kg | common | grammar.trailingApproxWords | Missing from pack. | + +### fuzzy-amount + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 几个 | a few (~3) | very-common | ENGINE | 几 not in fuzzyAmounts + CJK merge. | + +### range-bound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 三四个 | range 3-4 | very-common | ENGINE | Adjacent-number range; no separator. | +| 七八个 | range 7-8 | very-common | ENGINE | Same pattern. | +| 五公斤到十公斤 | range 5-10 kg | very-common | grammar.rangeSeparatorWords | Works with Arabic digits. | +| 三至五天 | range 3-5 days | common | ENGINE | 至 in rangeSeparatorWords but CJK numeral fails. | + +### unit-alias + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 三斤 | 1500g (斤=500g) | very-common | units.mass | Traditional unit; not in current aliases. | +| 二两肉 | 100g (两=50g) | very-common | units.mass | HAZARD: 两 is also numeral 2. Homograph. | +| 三里路 | 1500m (里=500m) | common | units.length | Traditional distance; 路 is filler. | +| 五亩地 | area (亩=666.67m2) | common | ENGINE | Area kind not in zh units. | + +### date-relative + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 前天 | dayOffset -2 | very-common | date.dayOffsets | Missing from pack. Data fix. | +| 后天 | dayOffset +2 | very-common | date.dayOffsets | Missing. Data fix. | +| 大前天 | dayOffset -3 | common | date.dayOffsets | Data fix. | +| 大后天 | dayOffset +3 | common | date.dayOffsets | Data fix. | +| 上周三 | last Wednesday | very-common | ENGINE | No-space modifier+weekday; needs CJK weekday parsing. | +| 下周一 | next Monday | very-common | ENGINE | Same. | +| 这周五 | this Friday | very-common | ENGINE | Same. | +| 周末 | this weekend | very-common | ENGINE | Weekend concept not in pack fields. | +| 月底 | end of month | very-common | ENGINE | No period-anchored date mechanism. | +| 月初 | beginning of month | common | ENGINE | Same. | +| 三个小时后 | 3h from now | very-common | ENGINE | 个 classifier blocks compactOffset matching. | +| 一个半小时后 | 1.5h from now | very-common | ENGINE | Classifier + half + offset compound. | +| 半天 | half a day | very-common | ENGINE | 半 as leading fraction; CJK merge. | +| 一刻钟 | 15 min | common | ENGINE | 刻钟 not in unit aliases. | +| 过三天 | in 3 days | common | date.compactOffset | 过 as future prefix; not in pack. | +| 再过两天 | in 2 more days | common | ENGINE | 再过 compound prefix; no multi-char prefix field. | +| 三天以后 | after 3 days | common | date.compactOffset | 以后 not in futureSuffixes. | +| 明年三月 | March next year | common | ENGINE | Relative year + month compound. | + +### date-absolute + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 7月15号 | July 15th | very-common | ENGINE | 号 as day-of-month marker not handled. | + +### time-of-day + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 下午三点半 | 15:30 | very-common | ENGINE | CJK time pattern not supported. | +| 三点一刻 | 3:15 | very-common | ENGINE | 一刻 = quarter. | +| 差一刻四点 | 3:45 | common | ENGINE | Subtractive time (差=minus). | +| 晚上八点 | 20:00 | very-common | ENGINE | Day-part prefix + time. | +| 凌晨三点 | 3:00 AM | common | ENGINE | 凌晨 = early morning. | +| 中午十二点 | 12:00 noon | common | ENGINE | 中午 prefix + CJK numeral. | + +--- + +## Japanese (ja) — 52 idioms + +Register landscape: no word boundaries in CJK text; postfix grammar particles; +kanji numeral composition shares structure with Chinese but has unique features +(counters/classifiers mandatory, hiragana readings as synonyms). Wave dash +(U+301C) and fullwidth tilde (U+FF5E) are standard range separators. +After-next/before-last modifiers (再来/先々) are productive. + +### number-word + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 三キロ | 3 kg | very-common | ENGINE | Single kanji + katakana unit merged as one token. | +| 三十五キロ | 35 kg | very-common | ENGINE | Compound kanji number + unit, one token. | +| 百五十グラム | 150 g | very-common | ENGINE | 百 scale + ones + unit in one token. | +| 千五百メートル | 1500 m | common | ENGINE | 千 scale; same tokenizer merge. | +| 三万五千円 | 35000 JPY | very-common | ENGINE | 万 scale composition; currency unit 円. | +| 3万5千キロ | 35000 kg | very-common | ENGINE | Mixed arabic + kanji scale. | +| 二億三千万 | 230000000 | common | ENGINE | 億 scale; multi-scale stacking. | +| 三点五キロ | 3.5 kg | common | ENGINE | 点 as decimal; all one token. | + +### fraction + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 二キロ半 | 2.5 kg | very-common | ENGINE | Post-unit 半; parses as 2 kg (半 lost). | +| 一時間半 | 1.5 h | very-common | ENGINE | N+unit+半 duration; one CJK token. | +| 半年 | 0.5 years | very-common | ENGINE | Leading 半 + unit; merged token. | +| 半日 | 0.5 days | very-common | ENGINE | Same pattern. | + +### approx-qualifier + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 五キロぐらい | ~5 kg | very-common | ENGINE | ぐらい merged with unit token. | +| 五キロくらい | ~5 kg | very-common | ENGINE | くらい variant; same merge. | +| 5キロほど | ~5 kg | very-common | grammar.trailingApproxWords | ほど not in any list; also CJK merge. | +| 十個前後 | ~10 (range 8-12) | common | ENGINE | 前後 implies +/- range; not expressible. | + +### fuzzy-amount + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 数個 | several (~3-5) | very-common | ENGINE | 数+counter pattern; merged token. | +| 十数キロ | 10-something | common | ENGINE | Teens-ish fuzzy pattern. | + +### range-bound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 三、四個 | range 3-4 | very-common | ENGINE | Adjacent-number with 、separator. | +| 5〜10キロ | range 5-10 | very-common | ENGINE | U+301C wave dash not normalized. | +| 三〜五日 | range 3-5 days | very-common | ENGINE | CJK numeral + wave dash. | +| 5～10キロ | range 5-10 | very-common | ENGINE | U+FF5E fullwidth tilde not normalized. | +| 5キロ以上 | at least 5 kg | very-common | ENGINE | 以上 merges with unit token ('キロ以上'). | +| 5キロ未満 | less than 5 kg | very-common | ENGINE | 未満 same CJK suffix merge. | + +### unit-alias + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 30坪 | 99.17 m2 (tsubo) | common | units (area) | Traditional area unit; needs kind registration. | +| 6畳 | 9.93 m2 (jo/tatami) | common | units (area) | Traditional area unit. | +| 一合 | 180 ml (go) | occasional | units.volume | Traditional volume. | + +### date-relative + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 一昨日 | dayOffset -2 | very-common | date.dayOffsets | Data fix. | +| おととい | dayOffset -2 | very-common | date.dayOffsets | Hiragana reading; data fix. | +| 明後日 | dayOffset +2 | very-common | date.dayOffsets | Data fix. | +| あさって | dayOffset +2 | very-common | date.dayOffsets | Hiragana reading; data fix. | +| しあさって | dayOffset +3 | occasional | date.dayOffsets | Regional (Kanto standard). | +| 再来週 | week after next | common | ENGINE | No 'after-next' modifier in DateRelativeModifier. | +| 先々週 | week before last | common | ENGINE | No 'before-last' modifier. | +| 再来月 | month after next | common | ENGINE | Same. | +| あと三日 | 3 days from now | very-common | ENGINE | あと as future PREFIX; compactOffset only has suffixes. | +| 三日以内 | within 3 days | very-common | ENGINE | 以内 (within) bounded-future; no field. | +| 一時間半後 | 1.5h from now | very-common | ENGINE | parseCompactNumber cannot handle 半 suffix. | +| 半年前 | 6 months ago | very-common | ENGINE | parseCompactNumber returns null for 半. | +| 今朝 | this morning ~8h | very-common | date.dayTimePhrases | Data fix. | +| 今夜 | tonight ~21h | very-common | date.dayTimePhrases | Data fix. | +| 今晩 | this evening ~19h | very-common | date.dayTimePhrases | Data fix. | +| 明日の朝 | tomorrow morning | very-common | ENGINE | の-joined date+time composition. | +| 週末 | this weekend | very-common | ENGINE | No weekend concept in pack fields. | +| 月末 | end of month | very-common | ENGINE | No period-anchored mechanism. | + +### time-of-day + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 午後3時半 | 15:30 | very-common | ENGINE | Japanese time format not in TIME_CORE_PATTERN. | +| 午前9時 | 09:00 | very-common | date.timePattern | Needs custom regex + parsing logic. | +| 15時半 | 15:30 | very-common | date.timePattern | 半 as :30 needs engine awareness. | +| 15時30分 | 15:30 | very-common | date.timePattern | N時N分 standard format. | +| 夜8時 | 20:00 | common | ENGINE | Day-part prefix + time. | + +### duration + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| 四半期 | 1 quarter (3 months) | common | ENGINE | Fixed compound word; not decomposable. | + +### compound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| りんご三個 | 3 apples | very-common | ENGINE | noun+number+counter; no counter concept. | + +--- + +## English (en) — 52 idioms + +Register landscape: base language with most mature pack. Remaining gaps are +mostly edge cases: colloquial elision ("one eighty"), pre-unit -ish, currency +multiplier words, UK date patterns, day+time-part compounds without "at", +and number-word clock hours. + +### approx-qualifier + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| twenty-odd | ~20 | very-common | grammar.trailingApproxWords | Data fix: add 'odd'. | +| thirty-something | ~30-39 | common | grammar.trailingApproxWords | Data fix: add 'something'. | +| 5-ish kg | ~5 kg | very-common | ENGINE | -ish between number and unit; unit-matcher fails. | +| 5 kg or thereabouts | ~5 kg | common | grammar.trailingApproxPhrases | Data fix: add 'or thereabouts'. | +| 5 lbs give or take | ~5 lbs | very-common | grammar.trailingApproxPhrases | Data fix: add 'give or take'. | +| ten-ish | ~10 | very-common | grammar.trailingApproxWords | Works unitless; fails as '10-ish minutes'. | +| pushing 60 | approaching 60 | common | ENGINE | No 'approaching from below' concept. | + +### range-bound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| north of 50 | >50 (exclusive lower) | common | grammar.boundPhrases | Data fix. | + +### number-word + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| one eighty | 180 (colloquial) | very-common | ENGINE | Parser computes 1+80=81; ambiguous without context. | +| a score | 20 | occasional | numberWords.scales | Data fix: add 'score':20. | +| 5k | 5000 | very-common | numberWords.scales | Partially works with kind hint; standalone fails. | + +### unit-alias (currency) + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| a grand | $1000 | very-common | ENGINE | Kind-scoped scale; cannot be in general scales. | +| five grand | $5000 | very-common | ENGINE | Same. | +| a buck fifty | $1.50 | common | numberWords | Works via typo-correction; 'buck' should be proper alias. | + +### compound + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| a few dozen | ~36-60 | common | ENGINE | fuzzyAmounts and dozenWords don't compose. | +| a good ten minutes | at least 10 min | very-common | ENGINE | No emphatic prefix phrase concept. | +| the better part of an hour | ~45-55 min | common | ENGINE | Multi-word template idiom. | + +### quantity-idiom + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| baker's dozen | 13 | occasional | ENGINE | Apostrophe-containing multi-word number. | +| couple-three | 2-3 (US regional) | occasional | ENGINE | Hyphenated number-word range. | + +### duration + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| a couple hours | ~2 h | very-common | ENGINE | Fuzzy amount returns range; duration rejects ranges. | +| quarter hour | 15 min | common | ENGINE | Bare fraction + unit without article. | + +### date-relative + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| tomorrow morning | tomorrow ~09:00 | very-common | ENGINE | dayOffset + time-part without 'at'. | +| yesterday evening | yesterday ~19:00 | very-common | ENGINE | Same compound issue. | +| Friday morning | Friday ~09:00 | very-common | ENGINE | Weekday + time-part compound. | +| next Monday morning | next Monday ~09:00 | very-common | ENGINE | Same. | +| this coming Friday | next Friday | very-common | date.modifiers | 'coming' as next synonym; 2-word modifier issue. | +| Monday week | Monday next week (UK) | common | ENGINE | Post-weekday 'week' offset; no field. | +| a week Monday | same (UK) | common | ENGINE | Duration + bare weekday without 'from'. | +| week on Tuesday | same (UK) | common | ENGINE | 'week on' + weekday. | +| Tuesday fortnight | Tuesday +14d (UK) | occasional | ENGINE | Same pattern with 'fortnight'. | +| the week after next | 2 weeks from now | common | ENGINE | Double-forward modifier. | +| the day after tomorrow | dayOffset +2 | very-common | date.dayOffsets | 'the' prefix not stripped; add to fillers. | +| early next week | Mon/Tue next week | very-common | ENGINE | No position qualifier for periods. | +| late July | ~July 25-31 | common | ENGINE | Same. | +| back end of the week | Thu/Fri (UK) | common | ENGINE | Colloquial position qualifier. | +| midweek | Wednesday | common | date.dayTimePhrases | Data fix or compound parse. | +| end of day | ~17:00 today | very-common | date.dayTimePhrases | Data fix. | +| close of business | ~17:00 | common | date.dayTimePhrases | Data fix. | +| first thing | ~08:00-09:00 | very-common | date.dayTimePhrases | Needs forward-date logic. | +| crack of dawn | ~05:30 | common | date.dayTimePhrases | Data fix. | +| this time next week | same time +7d | common | ENGINE | Time-preservation concept. | + +### time-of-day + +| input | meaning | freq | target | notes | +|-------|---------|------|--------|-------| +| quarter of five | 4:45 (US) | common | ENGINE | 'of' as 'to'; conflicts with 'quarter of an hour'. | +| five o'clock | 5:00 | very-common | ENGINE | Requires digit; doesn't resolve word numbers. | +| noon-ish | ~12:00 | common | ENGINE | -ish suffix on time alias not handled. | +| EOD | 17:00 | very-common | date.timeAliases | Data fix. | +| COB | 17:00 | very-common | date.timeAliases | Data fix. | +| half five | 5:30 (UK) | very-common | already works | No gap. | +| quarter past five | 5:15 | very-common | already works | No gap. | +| five till midnight | 23:55 | common | ENGINE | hourAmount() doesn't accept time aliases. | +| at 5 | 5:00 | very-common | ENGINE | Bare 'at' + digit; ambiguous meridiem. | +| Friday at five | Friday 5:00 | common | ENGINE | Word-number hour in time. | +| the fifteenth | 15th of this month | common | ENGINE | Bare ordinal without month. | + +--- + +## Cross-language engine gaps + +Evidence from `packages/lingo/src/` with file:line references. + +### 1. CJK multi-character number composition broken by tokenizer + +**Evidence:** `parse/tokenize.ts:97-100` (isCjkWord groups all adjacent CJK into +one token); `number/words.ts:143,150,171` (parser matches whole tokens against +ones/tens/scales) + +**Impact:** zh/ja: ALL multi-character number-word expressions fail (三百, 二万, +十五, 三百五十万 all produce NO_VALUE). Only single-char numbers that happen to be +one token work. This is the #1 blocker for CJK quantity parsing. + +**Proposal:** CJK number-word sub-parser that walks WITHIN a single CJK word +token, splitting it into (digit)(scale)(digit)(scale)... constituents. No new +pack field needed. Alternatively, tokenizer splits CJK runs at scale-word +boundaries using the loaded profile's scales keys. + +### 2. Trailing-approx particles (左右/ぐらい/-ish/mas o menos) not pack-expressible in suffix position + +**Evidence:** `parse/finish.ts:291-299` (only checks trailingApproxWords/ +trailingApproxPhrases); `tokenize.ts:97-100` (CJK suffix particles merged into +unit token: '公斤左右' is one token) + +**Impact:** zh/ja: 左右 and ぐらい are in approximateWords (prefix only) not +trailingApproxWords. Even if moved, tokenizer merges them with preceding unit. +es: 'mas o menos' as trailing phrase not expressible. All non-English trailing +approximation markers are blocked. + +**Proposal:** (1) Data-only for romance: add trailingApproxPhrases entries. (2) +Engine for CJK: unit-matcher strips known trailing-approx suffixes from CJK +tokens, OR tokenizer splits at suffix-particle boundaries. New optional field: +`trailingApproxSuffixes`. + +### 3. French vigesimal composition only works space-separated + +**Evidence:** `number/words.ts:150-163` (tens+hyphen+ones requires ones value < +10); `parse/tokenize.ts` (hyphen produces sym token) + +**Impact:** fr: 'soixante-dix' hyphenated fails because 'dix'=10 is not < 10. +quatre-vingt-dix (90) fails entirely (requires 4*20+10 multiplicative +composition). Currently misread as ranges. + +**Proposal:** Either exhaustive entries in `numberWords.composed` +(soixante-dix:70 through quatre-vingt-dix-neuf:99) via CLDR RBNF, or a new +`numberWords.compounds` table for multiplicative patterns. + +### 4. Spoken-clock-time grammar is hardcoded English + +**Evidence:** `date/time.ts:177-199` (TIME_NUM_WORDS hardcoded); +`time.ts:229-258` (parseRelativeMinutes uses English regex: +'past|after|to|till|til|before') + +**Impact:** All non-English: 'midi et quart' (fr), 'las tres menos cuarto' (es), +'duas e meia' (pt), 午後3時半 (zh/ja) all fail. The relative-minute grammar has +no locale extension point. + +**Proposal:** New optional DateVocabPack fields: `clockPastWords`, +`clockToWords`, `clockMinuteWords` (quarter->15, half->30), `clockHalfBefore` +(boolean for German-style 'halb 3' = 2:30). + +### 5. Adjacent-number ranges (三四个) not expressible for CJK + +**Evidence:** `parse/range.ts:137-192` (rangeSeparator requires separator token); +`tokenize.ts:97-100` (三四个 is one CJK token) + +**Impact:** zh/ja: idiomatic adjacent-number ranges cannot parse. Even spaced +forms need rangeAlternativeWords or separator mechanism. + +**Proposal:** (1) Data: add CJK rangeAlternativeWords. (2) Engine: CJK number +sub-parser detects adjacent-numeral patterns and emits implicit range. New +optional field: `adjacentNumberRange` (boolean). + +### 6. Unit-before-number order not supported + +**Evidence:** `parse/quantity.ts:279-310` (parseQty always parses value first) + +**Impact:** Minor: some informal Romance forms. Currency prefix is the only +exception and is already handled. + +**Proposal:** No change needed. Document as intentional constraint. + +### 7. 万/亿 scale grouping differs from Western composition + +**Evidence:** `number/words.ts:171-178` (scale handling algorithm) + +**Impact:** zh/ja: even if tokenizer gap is fixed, 三百五十万 (3,500,000) requires +(3*100 + 50) * 10000. The 十 in ones-table (value 10) conflicts with its +positional scale role. + +**Proposal:** New optional `NumberWordTables.positionalScales` field for scales +that multiply the current accumulator (十: 10) rather than promoting to total. + +--- + +## Field-coverage matrix + +Fields where non-English packs are empty or inherit semantically-wrong English +values. Source: all 7 locale packs vs engine consumption points. + +### Grammar fields (empty/inherits-en = needs locale data) + +| field | es | fr | pt | zh | ja | consumed by | +|-------|----|----|----|----|----|----| +| compoundJoinWords | empty | empty | empty | empty | empty | quantity.ts:410 | +| compoundMinusWords | empty | empty | empty | empty | empty | quantity.ts:416 | +| compoundPlusWords | empty | empty | empty | empty | empty | quantity.ts:413 | +| globalFillers | empty | empty | empty | empty | empty | finish.ts:206 | +| qualifierArticleFollowers | empty | empty | empty | empty | empty | quantity.ts:98 | +| qualifierArticleWords | empty | empty | empty | empty | empty | quantity.ts:99 | +| qualifierFillers | empty | empty | empty | empty | empty | quantity.ts:88 | +| qualifierSkipAfterApprox | empty | empty | empty | empty | empty | quantity.ts:125 | +| qualifierSoftenerPhrases | empty | empty | empty | empty | empty | quantity.ts:109 | +| qualifierSoftenerWords | empty | empty | empty | empty | empty | quantity.ts:104 | +| rangeAlternativeWords | empty | empty | empty | empty | empty | range.ts:147 | +| trailingApproxPhrases | empty | empty | empty | empty | empty | finish.ts:291 | +| trailingApproxWords | empty | empty | empty | empty | empty | finish.ts:297 | +| trailingOkWords | empty | empty | empty | empty | empty | finish.ts:302 | + +### Date vocab fields (empty/inherits-en or sparse) + +| field | es | fr | pt | zh | ja | consumed by | +|-------|----|----|----|----|----|----| +| weekdays | filled | filled | filled | empty | empty | relative.ts:285 | +| weekdayNames | filled | filled | filled | empty | empty | relative.ts:319 | +| unitWords | filled/16 | filled/16 | filled/14 | empty | empty | relative.ts:508 | +| dayTimePhrases | filled/6 | filled/6 | filled/8 | filled/2 | filled/2 | relative.ts:91 | +| timePattern | empty | empty | empty | empty | empty | parse.ts:260 | +| timeCorePattern | empty | empty | empty | empty | empty | parse.ts:201 | +| subunit | empty | empty | empty | empty | empty | relative.ts:550 | + +### Dead/unconsumed fields + +| field | notes | +|-------|-------| +| FuzzyVocab.fillers | Declared in `core/registry.ts:83` but NO engine code reads it. globalFillers in grammar serves this role via `finish.ts:206`. | +| LocalePack.numerals | Consumed ONLY by `parseCompactNumber` in `date/relative.ts:740-746` for CJK compact offsets. NOT consumed by the main number-word parser (`words.ts`). | + +--- + +## Prior art + +### What chrono-node, Duckling, Recognizers-Text, and CLDR teach + +**chrono-node** (MIT, 14 locales): imperative per-locale parsers + refiners +pipeline. Constants are flat dictionaries (string->number). LACKS data-driven +idiom coverage (no vigesimal French, no "en huit", no CJK scale composition). +Proves that imperative locale code does not scale. Japanese locale uses +normalizeTextToKanji() (hiragana -> kanji) before matching. The absence of these +idioms in a 14-locale library with active maintenance validates lingo's claim +that data-only packs done right can outperform hand-rolled parsers. + +**facebook/duckling** (BSD-3-Clause, 49 languages, 15 dimensions): compositional +rule engine + per-language Rules.hs + Corpus.hs. French handles 'en huit/quinze' +via pattern-matching rules. CJK numerals use scale-suffix composition rules. +Corpus.hs files are gold-standard (input, expected_value) test pairs. The +combination of compositional rules + validation corpus is the architecture to +emulate in data form. + +**Microsoft/Recognizers-Text** (MIT, 15 cultures): YAML-driven per-culture +pattern definitions. Most data-driven of the three. Key novel mechanisms: +**AmbiguityFiltersDict** (excludes '十足', '大陆' where numeral chars appear in +non-numeric words), **AllowListRegex** (200+ counter/classifier words for CJK), +gender-inflected number maps (doscientos/doscientas). Validates data-driven +design at scale. + +**CLDR** (Unicode License, 100+ locales): the definitive source for generating +pack data. RBNF for number-word spellout rules (vigesimal, CJK, gender); +dateFields for relative-date vocabulary; units for localized unit aliases with +plural forms. + +### Ranked data sources (with licenses) + +1. **CLDR RBNF XML** (`unicode-org/cldr`, `common/rbnf/*.xml`) — Unicode License + (ICU), MIT-compatible. Canonical number-word spellout rules for 100+ locales. + Use to generate exhaustive numberWords entries. + +2. **CLDR dateFields JSON** (`unicode-org/cldr-json`, + `cldr-dates-full/main/*/dateFields.json`) — Unicode License. Structured + relative-date vocabulary directly mappable to dayOffsets/calendarPeriodPhrases/ + relative fields. + +3. **CLDR units JSON** (`unicode-org/cldr-json`, + `cldr-units-full/main/*/units.json`) — Unicode License. Localized unit names + with plural forms. Can generate unitAliases entries per locale. + +4. **duckling Corpus.hs** (`facebook/duckling`, + `Duckling/*/Lang/Corpus.hs`) — BSD-3-Clause (permissive, MIT-compatible for + derived test data with copyright notice). Gold-standard test pairs across 49 + languages x 15 dimensions. Priority: ZH/JA Numeral, FR Time, EN/ZH Quantity. + +5. **duckling Rules.hs** (same repo/license) — rule patterns as idiom inventory + checklist. Useful for validation, not direct embedding. + +6. **Recognizers-Text YAML** (`microsoft/Recognizers-Text`, + `Patterns/*/[Lang]-Numbers.yaml`) — MIT License. AmbiguityFiltersDict, gender + maps, AllowListRegex. Directly reusable. + +7. **chrono-node constants** (`wanasit/chrono`, `src/locales/*/constants.ts`) — + MIT License. Modest dictionaries for cross-reference only. + +--- + +## Test infrastructure proposal + +**Current state:** ~22 locale test cases total. Main corpus is English-only (~280 +breadth + ~85 date rows). Existing locale tests do NOT exercise most grammar +fields for non-English. + +**Proposed per-locale corpus shape:** + +Each locale gets: +- `tests/corpus/locale--source.mjs` — exports `breadthRows`/`dateRows`/ + `dateRangeRows` with the same structure as the main corpus source. +- `tests/corpus/locale-.json` — checked-in snapshot (contract). +- Gate script accepts `--update` flag to regenerate after intentional changes. + +Row categories per locale: +1. Number-word cardinal/ordinal composition (ones, tens, hundreds, thousands, + locale-specific scales) +2. All grammar paths: ranges (entre X y Y), bounds (al menos X), conversions + (X a Y), compounds (dos metros y medio), trailing approx (mas o menos) +3. Date paths: day offsets, relative durations, weekday+modifier, calendar + periods, time aliases, compact offsets (CJK) +4. Known-failing idioms marked with `xfail` flag and target issue code, + graduating to pass as engine gaps are closed + +The gate runs in `bun run check`. New rows are ADDITIVE; xfail documents gaps +without blocking CI. From fc211940aac56ec09062b9ba79c4a69909f00609 Mon Sep 17 00:00:00 2001 From: Aymeric Rabot Date: Fri, 10 Jul 2026 01:11:58 +0200 Subject: [PATCH 3/3] chore: entropy pass, docs/llms sync, contribution readiness for 0.2.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dedupe parallel-agent redundancy: shared eatAnyPhrase (parse/config), canonical stripDateFillers (date/state), dead number-word re-exports removed (-17 net LOC, behavior pinned by 912 tests + 6 corpus contracts). - README/llms.txt/CONTRIBUTING synced to post-wave reality with execution-verified examples (treinta y cinco kilos, 三十五公斤, las tres menos cuarto, Monday week, 月底); site data refreshed. - Contribution surface at editor parity: YAML issue forms (bug/feature), blank issues off, Discussions enabled with Q&A/Ideas links. Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 4.8 Co-Authored-By: Codex GPT-5.5 --- .github/ISSUE_TEMPLATE/bug_report.md | 26 --------- .github/ISSUE_TEMPLATE/bug_report.yml | 65 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 +++ .github/ISSUE_TEMPLATE/feature_request.md | 22 -------- .github/ISSUE_TEMPLATE/feature_request.yml | 40 +++++++++++++ CONTRIBUTING.md | 9 +++ apps/site/public/llms-small.txt | 34 +++++++++-- packages/lingo/README.md | 42 +++++++++++--- packages/lingo/llms.txt | 30 ++++++++-- packages/lingo/src/date/absolute.ts | 14 +---- packages/lingo/src/date/relative.ts | 23 ++++---- packages/lingo/src/date/state.ts | 12 ++++ packages/lingo/src/number/words.ts | 7 --- packages/lingo/src/parse/config.ts | 11 ++++ packages/lingo/src/parse/finish.ts | 11 +--- packages/lingo/src/parse/quantity.ts | 11 +--- 16 files changed, 246 insertions(+), 119 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index df3ca4d..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -name: Bug report -about: A parse, conversion, format, or validation result that is wrong -labels: bug ---- - -## Input and options - -```ts -import { lingo } from '@pascal-app/lingo' -lingo('...', { /* options */ }) -``` - -## Expected - -What you expected the result (or issue codes) to be. - -## Actual - -The actual result. `JSON.stringify(result)` output is ideal — it carries the -schema version, spans, and issue codes. - -## Environment - -- `@pascal-app/lingo` version: -- Runtime (Node/bun/browser + version): diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..5e03472 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,65 @@ +name: 🐛 Bug report +description: A parse, conversion, format, or validation result that is wrong +labels: ["bug", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Thanks for the report! A concrete input + the serialized result is + usually all we need to reproduce a parser bug. + + - type: textarea + id: input + attributes: + label: Input and options + description: The exact call that misbehaves. + value: | + ```ts + import { lingo } from '@pascal-app/lingo' + lingo('...', { /* options */ }) + ``` + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected + description: What you expected the result (or issue codes) to be. + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual + description: > + The actual result. `JSON.stringify(result)` output is ideal — it + carries the schema version, spans, and issue codes. + validations: + required: true + + - type: input + id: version + attributes: + label: "@pascal-app/lingo version" + placeholder: 0.2.1 + validations: + required: true + + - type: input + id: runtime + attributes: + label: Runtime + description: Node/bun/browser + version + placeholder: Node 22, bun 1.3, Chrome 126… + validations: + required: false + + - type: textarea + id: additional + attributes: + label: Additional context + description: Locale packs loaded, related issues, anything else useful. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..f184278 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: 💬 Questions & Help + url: https://github.com/pascalorg/lingo/discussions/categories/q-a + about: Ask questions and get help from the community + - name: 💡 Ideas & Discussion + url: https://github.com/pascalorg/lingo/discussions/categories/ideas + about: Share ideas and discuss features before opening a formal request diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 0cc6c39..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Feature request -about: A new input form, unit, kind, API, or integration -labels: enhancement ---- - -## What should lingo understand or do? - -Concrete inputs and expected outputs are the fastest path to a decision: - -```text -"2 stone 4 lb" → quantity, mass, 14.5 kg -``` - -## Why - -The use case — form field, LLM tool argument, MCP server, etc. - -## Scope notes (optional) - -Anything you already know: prior art, ambiguity hazards ("oz" is mass and -volume), locale concerns. See `plans/` for how specs are written. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..bfbc97f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,40 @@ +name: ✨ Feature request +description: A new input form, unit, kind, locale idiom, API, or integration +labels: ["enhancement", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Concrete inputs and expected outputs are the fastest path to a + decision. For locale idioms, one real sentence a native speaker would + type is worth a page of description. + + - type: textarea + id: what + attributes: + label: What should lingo understand or do? + description: Concrete inputs → expected outputs. + placeholder: | + "2 stone 4 lb" → quantity, mass, 14.5 kg + "quinze jours" → duration, 2 weeks + validations: + required: true + + - type: textarea + id: why + attributes: + label: Why + description: The use case — form field, LLM tool argument, MCP server, etc. + validations: + required: true + + - type: textarea + id: scope + attributes: + label: Scope notes (optional) + description: > + Anything you already know: prior art, ambiguity hazards ("oz" is mass + and volume), locale concerns. See `plans/` for how specs are written + and `wiki/research/locale-idioms.md` for the idiom checklist. + validations: + required: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02562f5..8b6544b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,15 @@ new issue codes need tests for the code, its copy, and its typed `data` payload. Hostile-input coverage (unicode, RTL, zero-width, 50k-char strings) is expected for parser-facing changes. +### Locale corpora + +Locale idioms are corpus-gated per pack. Add rows in +`packages/lingo/tests/corpus/locale--source.mjs`, regenerate checked-in +contracts with `node scripts/corpus-diff.mjs --write` from `packages/lingo/`, +and rely on `bun run check` to enforce them. When adding idioms to a locale +pack, use `plans/033-locale-idiom-coverage.md` and +`wiki/research/locale-idioms.md` as the checklist before touching data. + ## Docs are part of done A user-visible change updates, in the same PR: TSDoc `@example` on new public diff --git a/apps/site/public/llms-small.txt b/apps/site/public/llms-small.txt index 28a5e7c..6576144 100644 --- a/apps/site/public/llms-small.txt +++ b/apps/site/public/llms-small.txt @@ -23,11 +23,12 @@ const range = parseRange("between 5 and 10 kg", { kind: "mass" }) - `quantity(value, unitRef, kind?)`, `convert(v, from, to)`, `tryConvert(v, from, to)`, `convertDelta(v, from, to)`. - `createLingo({ registry?, kinds?, messages?, fuzzy?, locales? }) → instance` for SSR/multi-tenant isolation and loaded locale packs. - `fromJSON(json)` rehydrates `toJSON()` output. +- Result guards: `isQuantity`, `isRange`, `isConversion`, `isNumber`; helpers: `firstError`, `candidateOf`, `formatIssue`. Options `{ kind?, unit?, currency?, locale?, system?: 'us'|'imperial'|'metric', numberFormat?: 'auto'|'dot-decimal'|'comma-decimal', strictness?: 'forgiving'|'confirm'|'strict', accept?, tolerance?, escalate?, messages?, profile?, registry? }`. -Quantity: `.value`, `.base`, `.kind`, `.to(unitRef)`, `.valueIn(u)`, `.convertDelta(u)`, `.toMinor()`, `.format()`, `.toBest()`, `.toJSON()`. -QuantityRange: `.minBase/.maxBase/.min()/.max()/.plusMinus/.fuzzy/.contains(q)/.widthIn(u)/.to(u)/.format()`. +Quantity: `.value`, `.base`, `.kind`, `.to(unitRef)`, `.valueIn(u)`, `.convertDelta(u)`, `.toMinor()`, `.format()`, `.toBest()`, `.toJSON()`. Literal `.to()` targets are same-kind checked at compile time; dynamic strings still validate at runtime. +QuantityRange: `.minBase/.maxBase/.min()/.max()/.plusMinus/.fuzzy/.contains(q)/.widthIn(u)/.to(u)/.format()`. Literal `.to()` targets get the same check. Kinds: length mass temperature duration volume area speed data data_rate flow_rate acceleration pressure energy force torque power frequency angle percent luminous_intensity luminous_flux illuminance luminance voltage current resistance charge substance concentration radiation_absorbed_dose radiation_equivalent_dose radioactivity currency. Bases: m, kg, K, s, m³, m², m/s, byte, bit/s, m³/s, m/s², Pa, J, N, N⋅m, W, Hz, rad, %, cd, lm, lx, cd/m², V, A, Ω, C, mol, mol/m³, Gy, Sv, Bq, and self-canonical ISO currency codes. @@ -37,10 +38,10 @@ Kinds: length mass temperature duration volume area speed data data_rate flow_ra import { completions } from "@pascal-app/lingo/complete" const items = completions("10 kg to 16", { kind: "mass", limit: 8 }) -// [{ text, result, confidence, source }] — source: parse|alternative|unit-ambiguity|unit-prefix|implied-unit +// [{ text, result, confidence, source }] — source: parse|alternative|unit-ambiguity|unit-prefix|implied-unit|range-implied|cross-kind|date ``` -Distinct from success `alternatives`, failure `candidate`, and issue `suggestions`. Wire into `lingoInput({ complete, onComplete })` for autocomplete dropdowns. +Distinct from success `alternatives`, failure `candidate`, and issue `suggestions`. Wire into `lingoInput({ complete, onComplete })` for autocomplete dropdowns. Pass `date: (text) => parseDate(text, { now })` to opt into date completions without bundling `@pascal-app/lingo/date` into `@pascal-app/lingo/complete`. ## Locales (`@pascal-app/lingo/locales/*`) @@ -58,6 +59,27 @@ lingo.parseQuantity("5公斤", { locale: "zh" }) Packs: `en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`. Omit `locale` to detect among loaded packs plus English; pass `locale` for known fields. English is built in and does not require a pack. `parseDate(text, { locale, localePacks, now })` can use loaded packs for relative date vocabulary. +Pack notes: +- `en`: built-in English quantity/date grammar; wave-1 adds spoken decimals and date idioms such as `two point five kg` and `quarter of five`. +- `en-gb`: English plus day-first numeric dates and UK weekday offsets (`Monday week`, `Tuesday fortnight`). +- `es`: Spanish number/range words, Romance composition, spoken decimals, approximants, and date clock/edge idioms. +- `fr`: French number/range words, exhaustive CLDR-style vigesimal compounds, spoken decimals, approximants, and weekday-offset/clock idioms. +- `pt`: Portuguese number/range words, compound hundreds/tens, spoken decimals, approximants, and clock/day-offset idioms. +- `zh`: Chinese CJK number walker, scale grouping, post-unit half, duration counters, period edges, and day offsets. CJK spoken clock (`下午3点半`) is deferred. +- `ja`: Japanese CJK number walker, scale grouping, post-unit half, duration counters, wave-dash ranges, and double-step period modifiers. CJK spoken clock (`午後3時半`) is deferred. + +Verified locale examples (fixed `now = 2026-07-03 14:30` local time for date examples): + +```txt +en: "two point five kg" → 2.5 kg; "quarter of five" → 2026-07-04 04:45 +en-gb: "Monday week" → 2026-07-13; "Tuesday fortnight" → 2026-07-21 +es: "treinta y cinco kilos" → 35 kg; "dos coma cinco kg" → 2.5 kg; "las tres menos cuarto" → 2026-07-04 02:45 +fr: "quatre-vingt-dix kg" → 90 kg; "deux virgule cinq kg" → 2.5 kg; "deux heures et quart" → 2026-07-04 02:15; "lundi en huit" → 2026-07-13 +pt: "vinte e cinco kg" → 25 kg; "dois virgula cinco kg" → 2.5 kg; "quinze para as tres" → 2026-07-04 02:45; "depois de amanha" → 2026-07-05 +zh: "三十五公斤" → 35 kg; "两公斤半" → 2.5 kg; "七八天" → duration range 7..8 d; "月底" → 2026-07-31 +ja: "三十五キロ" → 35 kg; "一時間半" → 1.5 h; "5〜10キロ" → 5..10 kg; "再来週" → week of 2026-07-13 +``` + ## Dates (`@pascal-app/lingo/date`) ```ts @@ -88,7 +110,7 @@ field.set("6ft") field.commit() // hidden input submits canonical value ``` -`field.value` · `.quantity` · `.result` · `.state` ('idle'|'incomplete'|'valid'|'invalid') · `.set()` · `.commit()` · `.update()` · `.destroy()`. Never rewrites while typing; canonicalizes on blur/Enter/submit. React: `useLingoInput(opts)` from `@pascal-app/lingo/react`. +`field.value` · `.quantity` · `.result` · `.state` ('idle'|'incomplete'|'valid'|'invalid') · `.set()` · `.commit()` · `.update()` · `.destroy()`. Never rewrites while typing; canonicalizes on blur/Enter/submit. With `complete`/`onComplete`, the input gets the headless WAI-ARIA combobox attributes (`role="combobox"`, `aria-autocomplete="list"`, `aria-expanded`); pass `listboxId` to set `aria-controls` for your rendered listbox. React: `useLingoInput(opts)` from `@pascal-app/lingo/react`. ## Element (`@pascal-app/lingo/element`) @@ -201,7 +223,7 @@ The `/docs#forms-ux` section and `/docs/forms-ux.md` markdown mirror show the sa | RANGE_MIN / RANGE_MAX | Value outside bounds | Re-emit within min/max advertised in field description | | RATE_REQUIRED | Cross-currency without rates | Call `convertCurrency` with injected rates | -Full list: RANGE_KIND_MISMATCH, CONVERSION_KIND_MISMATCH, TRAILING_INPUT, SINGLE_VALUE_EXPECTED, APPROX_NOT_ALLOWED, NUMBER_FORMAT, NONFINITE, LOCALE_NOT_LOADED, RANGE_OPEN_BOUND_NOT_ALLOWED, REQUIRED, UNSUPPORTED_DATE, AMBIGUOUS_DATE, RANGE_REVERSED, COMPOUND_OVERFLOW, CIVIL_AVERAGE, UNIT_ASSUMED, WEEKDAY_ASSUMED_NEXT, SLANG_UNIT, AMBIGUOUS_TIMEZONE. Override copy via `messages` option map. +Full list: EMPTY, NO_VALUE, UNKNOWN_UNIT, KIND_MISMATCH, RANGE_KIND_MISMATCH, CONVERSION_KIND_MISMATCH, RATE_REQUIRED, TRAILING_INPUT, SINGLE_VALUE_EXPECTED, APPROX_NOT_ALLOWED, UNIT_REQUIRED, CONVERSION_NOT_ALLOWED, NUMBER_FORMAT, NONFINITE, LOCALE_NOT_LOADED, RANGE_MIN, RANGE_MAX, RANGE_OPEN_BOUND_NOT_ALLOWED, REQUIRED, UNSUPPORTED_DATE, NOW_REQUIRED, TYPO_CORRECTED, AMBIGUOUS_NUMBER, AMBIGUOUS_UNIT, AMBIGUOUS_DATE, RANGE_REVERSED, COMPOUND_OVERFLOW, CIVIL_AVERAGE, UNIT_ASSUMED, WEEKDAY_ASSUMED_NEXT, SLANG_UNIT, TZ_IGNORED, AMBIGUOUS_TIMEZONE. Override copy via `messages` option map. ## Canonical examples (input → essence) diff --git a/packages/lingo/README.md b/packages/lingo/README.md index 83c6f41..7bc5493 100644 --- a/packages/lingo/README.md +++ b/packages/lingo/README.md @@ -8,9 +8,11 @@ humanizes them back. Zero runtime dependencies. The size gate lives in `scripts/size.mjs`, and the package includes thirty-three built-in kinds (chrono-node needs 35 kB for dates alone). English is the default; opt-in locale packs (`createLingo({ locales })`, `@pascal-app/lingo/locales/*`) add number words, -grammar, units, and relative dates for Spanish, French, Portuguese, Chinese, -Japanese, and en-GB (plan 031). Number *formatting* is locale-aware via `Intl` in -every locale; `humanizeDate()` output stays English for now. +grammar, units, and relative-date idioms for Spanish, French, Portuguese, +Chinese, Japanese, and en-GB. Wave-1 packs cover Romance number composition, +CJK number tokens, localized date grammar, and per-locale corpus gates. Number +*formatting* is locale-aware via `Intl` in every locale; `humanizeDate()` output +stays English for now. ```ts import { lingo, parseQuantity, convert, tryConvert } from '@pascal-app/lingo' @@ -259,6 +261,9 @@ lingo('1 kgf/cm²').quantity.to('kPa').value // 98.0665 — use kgf/cm²; k quantity(3e5, 'm').format({ notation: 'scientific' }) // "3e5 m" (also 'engineering'; every style re-parses) ``` +Literal `Quantity.to()` / `QuantityRange.to()` targets are same-kind checked +too; dynamic strings still validate at runtime. + Bare `bps` is finance shorthand for basis points. Use `bit/s`, `kbit/s`, or network spellings such as `Mbps` when you mean bits per second. Untyped glued `1M` stays rejected because `M` is a future multiplier hazard; @@ -447,7 +452,9 @@ completions('5', { kind: 'length' }).map((c) => c.text) The DOM field is headless about this too: `lingoInput` accepts injected `complete` / `onComplete` hooks, so you render your own dropdown — the library ships -no UI. +no UI. When completions are enabled it wires the combobox side of the ARIA +contract (`role="combobox"`, `aria-autocomplete="list"`, `aria-expanded`), and +`listboxId` sets `aria-controls` for your listbox. ## Extending @@ -487,18 +494,37 @@ Parse non-English input by loading packs on an instance — English stays the de ```ts import { createLingo } from '@pascal-app/lingo' +import { parseDate } from '@pascal-app/lingo/date' import { es } from '@pascal-app/lingo/locales/es' +import { zh } from '@pascal-app/lingo/locales/zh' -const app = createLingo({ locales: [es] }) -app.parse('dos kg', { locale: 'es' }) // 2 kg +const app = createLingo({ locales: [es, zh] }) +app.parse('dos kg', { locale: 'es' }) // 2 kg app.parse('al menos 2 m', { locale: 'es' }) // ≥ 2 m (open range) app.parse('entre 5 y 10 kg', { locale: 'es' }) // 5–10 kg + +const spanish = app.parseQuantity('treinta y cinco kilos', { locale: 'es' }) +spanish.ok && spanish.quantity.to('kg').value // 35 + +const chinese = app.parseQuantity('三十五公斤', { locale: 'zh' }) +chinese.ok && chinese.quantity.to('kg').value // 35 + +const clock = parseDate('las tres menos cuarto', { + locale: 'es', + localePacks: [es], + now: new Date(2026, 6, 3, 14, 30), +}) +clock.ok && [clock.date.getHours(), clock.date.getMinutes()] // [2, 45] + // omit `locale` to auto-detect among the loaded packs ``` Packs (`en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`) are additive and tree-shakeable; they add number words, units, ranges, and relative dates (through -`@pascal-app/lingo/date`) for their language. Requesting an unloaded locale returns +`@pascal-app/lingo/date`) for their language. The first idiom wave includes +Romance tens/hundreds/decimal words, CJK scale/grouped numbers and post-unit +half, localized clock phrases, period edges, weekday offsets, and locale unit +words in date/duration parsing. Requesting an unloaded locale returns `LOCALE_NOT_LOADED` rather than silently parsing as English; `humanizeDate()` output stays English for now. @@ -784,7 +810,7 @@ NONFINITE · LOCALE_NOT_LOADED · RANGE_MIN · RANGE_MAX · REQUIRED · UNSUPPOR RANGE_OPEN_BOUND_NOT_ALLOWED · TYPO_CORRECTED · AMBIGUOUS_NUMBER · AMBIGUOUS_UNIT · AMBIGUOUS_DATE · RANGE_REVERSED · COMPOUND_OVERFLOW · CIVIL_AVERAGE · UNIT_ASSUMED · WEEKDAY_ASSUMED_NEXT · SLANG_UNIT · TZ_IGNORED · -AMBIGUOUS_TIMEZONE · LOCALE_NOT_LOADED` +AMBIGUOUS_TIMEZONE` Every issue carries a typed `data` payload (`LingoIssue<'UNKNOWN_UNIT'>` knows `data.unit` and `data.suggestions`). `NOW_REQUIRED` fires when a date input diff --git a/packages/lingo/llms.txt b/packages/lingo/llms.txt index a99a635..6576144 100644 --- a/packages/lingo/llms.txt +++ b/packages/lingo/llms.txt @@ -23,11 +23,12 @@ const range = parseRange("between 5 and 10 kg", { kind: "mass" }) - `quantity(value, unitRef, kind?)`, `convert(v, from, to)`, `tryConvert(v, from, to)`, `convertDelta(v, from, to)`. - `createLingo({ registry?, kinds?, messages?, fuzzy?, locales? }) → instance` for SSR/multi-tenant isolation and loaded locale packs. - `fromJSON(json)` rehydrates `toJSON()` output. +- Result guards: `isQuantity`, `isRange`, `isConversion`, `isNumber`; helpers: `firstError`, `candidateOf`, `formatIssue`. Options `{ kind?, unit?, currency?, locale?, system?: 'us'|'imperial'|'metric', numberFormat?: 'auto'|'dot-decimal'|'comma-decimal', strictness?: 'forgiving'|'confirm'|'strict', accept?, tolerance?, escalate?, messages?, profile?, registry? }`. -Quantity: `.value`, `.base`, `.kind`, `.to(unitRef)`, `.valueIn(u)`, `.convertDelta(u)`, `.toMinor()`, `.format()`, `.toBest()`, `.toJSON()`. -QuantityRange: `.minBase/.maxBase/.min()/.max()/.plusMinus/.fuzzy/.contains(q)/.widthIn(u)/.to(u)/.format()`. +Quantity: `.value`, `.base`, `.kind`, `.to(unitRef)`, `.valueIn(u)`, `.convertDelta(u)`, `.toMinor()`, `.format()`, `.toBest()`, `.toJSON()`. Literal `.to()` targets are same-kind checked at compile time; dynamic strings still validate at runtime. +QuantityRange: `.minBase/.maxBase/.min()/.max()/.plusMinus/.fuzzy/.contains(q)/.widthIn(u)/.to(u)/.format()`. Literal `.to()` targets get the same check. Kinds: length mass temperature duration volume area speed data data_rate flow_rate acceleration pressure energy force torque power frequency angle percent luminous_intensity luminous_flux illuminance luminance voltage current resistance charge substance concentration radiation_absorbed_dose radiation_equivalent_dose radioactivity currency. Bases: m, kg, K, s, m³, m², m/s, byte, bit/s, m³/s, m/s², Pa, J, N, N⋅m, W, Hz, rad, %, cd, lm, lx, cd/m², V, A, Ω, C, mol, mol/m³, Gy, Sv, Bq, and self-canonical ISO currency codes. @@ -58,6 +59,27 @@ lingo.parseQuantity("5公斤", { locale: "zh" }) Packs: `en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`. Omit `locale` to detect among loaded packs plus English; pass `locale` for known fields. English is built in and does not require a pack. `parseDate(text, { locale, localePacks, now })` can use loaded packs for relative date vocabulary. +Pack notes: +- `en`: built-in English quantity/date grammar; wave-1 adds spoken decimals and date idioms such as `two point five kg` and `quarter of five`. +- `en-gb`: English plus day-first numeric dates and UK weekday offsets (`Monday week`, `Tuesday fortnight`). +- `es`: Spanish number/range words, Romance composition, spoken decimals, approximants, and date clock/edge idioms. +- `fr`: French number/range words, exhaustive CLDR-style vigesimal compounds, spoken decimals, approximants, and weekday-offset/clock idioms. +- `pt`: Portuguese number/range words, compound hundreds/tens, spoken decimals, approximants, and clock/day-offset idioms. +- `zh`: Chinese CJK number walker, scale grouping, post-unit half, duration counters, period edges, and day offsets. CJK spoken clock (`下午3点半`) is deferred. +- `ja`: Japanese CJK number walker, scale grouping, post-unit half, duration counters, wave-dash ranges, and double-step period modifiers. CJK spoken clock (`午後3時半`) is deferred. + +Verified locale examples (fixed `now = 2026-07-03 14:30` local time for date examples): + +```txt +en: "two point five kg" → 2.5 kg; "quarter of five" → 2026-07-04 04:45 +en-gb: "Monday week" → 2026-07-13; "Tuesday fortnight" → 2026-07-21 +es: "treinta y cinco kilos" → 35 kg; "dos coma cinco kg" → 2.5 kg; "las tres menos cuarto" → 2026-07-04 02:45 +fr: "quatre-vingt-dix kg" → 90 kg; "deux virgule cinq kg" → 2.5 kg; "deux heures et quart" → 2026-07-04 02:15; "lundi en huit" → 2026-07-13 +pt: "vinte e cinco kg" → 25 kg; "dois virgula cinco kg" → 2.5 kg; "quinze para as tres" → 2026-07-04 02:45; "depois de amanha" → 2026-07-05 +zh: "三十五公斤" → 35 kg; "两公斤半" → 2.5 kg; "七八天" → duration range 7..8 d; "月底" → 2026-07-31 +ja: "三十五キロ" → 35 kg; "一時間半" → 1.5 h; "5〜10キロ" → 5..10 kg; "再来週" → week of 2026-07-13 +``` + ## Dates (`@pascal-app/lingo/date`) ```ts @@ -88,7 +110,7 @@ field.set("6ft") field.commit() // hidden input submits canonical value ``` -`field.value` · `.quantity` · `.result` · `.state` ('idle'|'incomplete'|'valid'|'invalid') · `.set()` · `.commit()` · `.update()` · `.destroy()`. Never rewrites while typing; canonicalizes on blur/Enter/submit. React: `useLingoInput(opts)` from `@pascal-app/lingo/react`. +`field.value` · `.quantity` · `.result` · `.state` ('idle'|'incomplete'|'valid'|'invalid') · `.set()` · `.commit()` · `.update()` · `.destroy()`. Never rewrites while typing; canonicalizes on blur/Enter/submit. With `complete`/`onComplete`, the input gets the headless WAI-ARIA combobox attributes (`role="combobox"`, `aria-autocomplete="list"`, `aria-expanded`); pass `listboxId` to set `aria-controls` for your rendered listbox. React: `useLingoInput(opts)` from `@pascal-app/lingo/react`. ## Element (`@pascal-app/lingo/element`) @@ -201,7 +223,7 @@ The `/docs#forms-ux` section and `/docs/forms-ux.md` markdown mirror show the sa | RANGE_MIN / RANGE_MAX | Value outside bounds | Re-emit within min/max advertised in field description | | RATE_REQUIRED | Cross-currency without rates | Call `convertCurrency` with injected rates | -Full list: RANGE_KIND_MISMATCH, CONVERSION_KIND_MISMATCH, TRAILING_INPUT, SINGLE_VALUE_EXPECTED, APPROX_NOT_ALLOWED, NUMBER_FORMAT, NONFINITE, LOCALE_NOT_LOADED, RANGE_OPEN_BOUND_NOT_ALLOWED, REQUIRED, UNSUPPORTED_DATE, AMBIGUOUS_DATE, RANGE_REVERSED, COMPOUND_OVERFLOW, CIVIL_AVERAGE, UNIT_ASSUMED, WEEKDAY_ASSUMED_NEXT, SLANG_UNIT, AMBIGUOUS_TIMEZONE. Override copy via `messages` option map. +Full list: EMPTY, NO_VALUE, UNKNOWN_UNIT, KIND_MISMATCH, RANGE_KIND_MISMATCH, CONVERSION_KIND_MISMATCH, RATE_REQUIRED, TRAILING_INPUT, SINGLE_VALUE_EXPECTED, APPROX_NOT_ALLOWED, UNIT_REQUIRED, CONVERSION_NOT_ALLOWED, NUMBER_FORMAT, NONFINITE, LOCALE_NOT_LOADED, RANGE_MIN, RANGE_MAX, RANGE_OPEN_BOUND_NOT_ALLOWED, REQUIRED, UNSUPPORTED_DATE, NOW_REQUIRED, TYPO_CORRECTED, AMBIGUOUS_NUMBER, AMBIGUOUS_UNIT, AMBIGUOUS_DATE, RANGE_REVERSED, COMPOUND_OVERFLOW, CIVIL_AVERAGE, UNIT_ASSUMED, WEEKDAY_ASSUMED_NEXT, SLANG_UNIT, TZ_IGNORED, AMBIGUOUS_TIMEZONE. Override copy via `messages` option map. ## Canonical examples (input → essence) diff --git a/packages/lingo/src/date/absolute.ts b/packages/lingo/src/date/absolute.ts index 5692b49..16dce78 100644 --- a/packages/lingo/src/date/absolute.ts +++ b/packages/lingo/src/date/absolute.ts @@ -1,7 +1,7 @@ import type { LingoIssue } from '../core/types' import { parseYear, sameDay, startOfDay, validDateTime } from './civil' import type { DateAlternative, DateGrain } from './parse' -import { type CoreDate, core, issue, knownFor, needsNow, type P } from './state' +import { type CoreDate, core, issue, knownFor, needsNow, type P, stripDateFillers } from './state' import { MONTHS as EN_MONTHS } from './vocab' export function parseAbsolute(p: P, start: number, end: number): CoreDate | null { @@ -151,18 +151,6 @@ function dateMonths(p: P): Record { return p.profile.date?.months ?? EN_MONTHS } -function stripDateFillers(p: P, source: string): string { - const fillers = p.profile.date?.fillerWords - if (!(fillers && fillers.length > 0)) { - return source - } - const fillerSet = new Set(fillers) - return source - .split(/\s+/) - .filter((word) => !fillerSet.has(word.toLowerCase())) - .join(' ') -} - function buildYearless(p: P, month: number, day: number, year: number | undefined): Date | null { let y = year ?? p.now.getFullYear() let date = validDateTime(y, month, day, 0, 0, 0) diff --git a/packages/lingo/src/date/relative.ts b/packages/lingo/src/date/relative.ts index fd49f74..0f8402a 100644 --- a/packages/lingo/src/date/relative.ts +++ b/packages/lingo/src/date/relative.ts @@ -15,7 +15,16 @@ import { withTime, } from './civil' import type { DateAlternative, DateGrain } from './parse' -import { type CoreDate, core, issue, knownFor, needsNow, type P, trimRange } from './state' +import { + type CoreDate, + core, + issue, + knownFor, + needsNow, + type P, + stripDateFillers, + trimRange, +} from './state' import { type DayTimePhrase, DAY_OFFSETS as EN_DAY_OFFSETS, @@ -845,18 +854,6 @@ function isPrefixAt(p: P, first: number, phrases: readonly string[] | undefined) return eatPrefix(p, first, phrases) > first } -function stripDateFillers(p: P, source: string): string { - const fillers = p.profile.date?.fillerWords ?? ['the'] - if (fillers.length === 0) { - return source - } - const fillerSet = new Set(fillers) - return source - .split(/\s+/) - .filter((word) => !fillerSet.has(word.toLowerCase())) - .join(' ') -} - function matchPeriodModifier( p: P, source: string, diff --git a/packages/lingo/src/date/state.ts b/packages/lingo/src/date/state.ts index a957f84..429f0ae 100644 --- a/packages/lingo/src/date/state.ts +++ b/packages/lingo/src/date/state.ts @@ -83,6 +83,18 @@ export function trimRange(text: string, start: number, end: number): Span { return { start, end } } +export function stripDateFillers(p: P, source: string): string { + const fillers = p.profile.date?.fillerWords ?? ['the'] + if (fillers.length === 0) { + return source + } + const fillerSet = new Set(fillers) + return source + .split(/\s+/) + .filter((word) => !fillerSet.has(word.toLowerCase())) + .join(' ') +} + export function knownFor(grain: DateGrain): string[] { if (grain === 'year') { return ['year'] diff --git a/packages/lingo/src/number/words.ts b/packages/lingo/src/number/words.ts index a6376e9..2163056 100644 --- a/packages/lingo/src/number/words.ts +++ b/packages/lingo/src/number/words.ts @@ -8,13 +8,6 @@ import type { Token } from '../parse/tokenize' * hyphenated compounds arrive as word/sym/word triples. */ -export const ONES: Record = EN_NUMBER_WORDS.ones -export const TENS: Record = EN_NUMBER_WORDS.tens -export const SCALES: Record = EN_NUMBER_WORDS.scales -/** value + spread for approximate amount words. */ -export const FUZZY_AMOUNTS: Record = - EN_NUMBER_WORDS.fuzzyAmounts - export interface WordNumberResult { approximate?: boolean /** diff --git a/packages/lingo/src/parse/config.ts b/packages/lingo/src/parse/config.ts index 43304ae..aa01395 100644 --- a/packages/lingo/src/parse/config.ts +++ b/packages/lingo/src/parse/config.ts @@ -385,6 +385,17 @@ export function eatPhrase(p: ParserState, i: number, phrase: string): number { return pos } +/** Try each phrase in order; return next token index after first match, or -1. */ +export function eatAnyPhrase(p: ParserState, pos: number, phrases: Iterable): number { + for (const phrase of phrases) { + const nx = eatPhrase(p, pos, phrase) + if (nx >= 0) { + return nx + } + } + return -1 +} + /** First token index whose start ≥ normalized position `pos`. Binary search. */ export function tokenAfter(p: ParserState, pos: number): number { let lo = 0 diff --git a/packages/lingo/src/parse/finish.ts b/packages/lingo/src/parse/finish.ts index 7f7c05f..c9858a4 100644 --- a/packages/lingo/src/parse/finish.ts +++ b/packages/lingo/src/parse/finish.ts @@ -4,6 +4,7 @@ import { Quantity, QuantityRange } from '../core/quantity' import { applySeverity, confidenceForIssues, + eatAnyPhrase, eatPhrase, exampleFor, type FailResult, @@ -266,16 +267,6 @@ function tryFuzzy(p: ParserState, i: number): Parsed | null { const TRAILING_OK = new Set(['.', '!', '?', ')', ',']) -function eatAnyPhrase(p: ParserState, pos: number, phrases: Iterable): number { - for (const phrase of phrases) { - const nx = eatPhrase(p, pos, phrase) - if (nx >= 0) { - return nx - } - } - return -1 -} - function finishTrailing(p: ParserState, parsed: Parsed): LingoResult { if (!parsed.result.ok) { return parsed.result diff --git a/packages/lingo/src/parse/quantity.ts b/packages/lingo/src/parse/quantity.ts index c0863c1..7d5f5e4 100644 --- a/packages/lingo/src/parse/quantity.ts +++ b/packages/lingo/src/parse/quantity.ts @@ -5,6 +5,7 @@ import type { Kind, UnitDef } from '../core/types' import { parseValue, type ValueCtx, type ValueNode } from '../number/value' import { parseAndFractionTail } from '../number/words' import { + eatAnyPhrase, eatPhrase, issue, type ParserState, @@ -45,16 +46,6 @@ function startsBound(p: ParserState, pos: number): boolean { return false } -function eatAnyPhrase(p: ParserState, pos: number, phrases: Iterable): number { - for (const phrase of phrases) { - const nx = eatPhrase(p, pos, phrase) - if (nx >= 0) { - return nx - } - } - return -1 -} - export function parseQualifiers(p: ParserState, i: number): Quals { let pos = i let approximate = false