diff --git a/ROADMAP.md b/ROADMAP.md index e3fa366..1aa1baf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -70,6 +70,8 @@ Status: βœ… complete Β· 🟑 in progress Β· ⬜ planned Β· β›” blocked - ⬜ User-selected ambiguous date format. - βœ… Numeric tolerance: absolute and percentage. - βœ… Deterministic column mapping suggestions without AI. +- βœ… Conservative key-confidence margin prevents silent auto-selection when multiple identifier keys are equally plausible. +- βœ… Known semantic groups are not auto-paired across conflicting meanings. - 🟑 Basic mapping validation exists; error UX still needs hardening. **Exit criteria:** mappings are explicit, auditable and never silently guess ambiguous formats. @@ -105,6 +107,8 @@ Status: βœ… complete Β· 🟑 in progress Β· ⬜ planned Β· β›” blocked - βœ… Node test coverage for parser and browser comparison service. - βœ… Synthetic behavior fixtures for real comparison outcomes. +- βœ… Negative parser/mapping tests cover malformed quotes, corrupt UTF-8 markers, duplicate headers, empty files, ambiguous keys and semantic conflicts. +- βœ… CSV export regression coverage protects formula-like key cells. - ⬜ Integration tests. - ⬜ Regression tests that fail on the pre-fix behavior. - ⬜ Accessibility audit: keyboard, labels, focus, screen reader semantics. @@ -147,8 +151,9 @@ Status: βœ… complete Β· 🟑 in progress Β· ⬜ planned Β· β›” blocked | 2026-09-26 | R4 β€” Visual Diff pivot | βœ… | Product flow changed to Upload β†’ automatic comparison β†’ side-by-side Visual Diff. Manual mapping moved behind Expert settings. | | 2026-09-26 | R5 β€” Real browser engine MVP | βœ… | Replaced mock default with real local CSV/TSV parsing, automatic mapping suggestions and deterministic comparison sufficient for the Visual Diff flow. Regression fixtures pass (4/4) and production build verified locally. | | 2026-09-26 | R6 β€” Repository polish | βœ… | Reworked EN/LV README around the Visual Diff product promise, documented strict project scope, strengthened contribution/security guidance, added GitHub issue/PR templates, README hero artwork and CI workflow. | -| 2026-09-26 | R7 β€” Alpha release preparation | 🟑 | Prepared v0.1.0-alpha.1 metadata, changelog and release notes; simplified CI to one PR check plus one main-branch check and added concurrency cancellation. Social Preview upload and tagged prerelease remain. | +| 2026-09-26 | R7 β€” Alpha release preparation | βœ… | Prepared and published v0.1.0-alpha.1 prerelease metadata, changelog and release notes; simplified CI to one PR check plus one main-branch check and added concurrency cancellation. Social Preview remains deferred until repository publication. | +| 2026-09-26 | R8 β€” Automatic matching hardening | βœ… | Added explicit mapping roles, conservative key ambiguity rejection, semantic conflict guards and broader parser/normalization/export regression coverage without changing the primary workflow. | ## Next target / Nākamais mΔ“rΔ·is -**Next:** publish the Social Preview, tag `v0.1.0-alpha.1` as a prerelease, then continue matching/normalization hardening without expanding the core workflow. +**Next:** conservative date-format handling, accessibility/security review and measured large-file behavior β€” without expanding the core workflow. diff --git a/src/app/app-controller.js b/src/app/app-controller.js index 86cb463..cfd308d 100644 --- a/src/app/app-controller.js +++ b/src/app/app-controller.js @@ -170,7 +170,19 @@ export class AppController { return; } - const [keySuggestion, ...comparisonSuggestions] = suggestions; + const keySuggestion = suggestions.find((suggestion) => suggestion.role === 'key') + ?? suggestions[0]; + + if (!keySuggestion) { + this.store.patch({ mapping: null, mappingValid: false }); + this.store.setState('MAPPING'); + return; + } + + const comparisonSuggestions = suggestions.filter((suggestion) => ( + suggestion !== keySuggestion && suggestion.role !== 'key' + )); + const mapping = { keys: [{ columnA: keySuggestion.columnA, diff --git a/src/engine/mapping-suggester.js b/src/engine/mapping-suggester.js index 48dee29..2cc14a2 100644 --- a/src/engine/mapping-suggester.js +++ b/src/engine/mapping-suggester.js @@ -1,3 +1,6 @@ +const MIN_KEY_SCORE = 0.55; +const MIN_KEY_MARGIN = 0.08; + export function suggestColumnMappings(fileA, fileB, dataA, dataB) { const columnsA = fileA.columns ?? []; const columnsB = fileB.columns ?? []; @@ -8,6 +11,7 @@ export function suggestColumnMappings(fileA, fileB, dataA, dataB) { if (!keyCandidate) return []; const suggestions = [{ + role: 'key', columnA: keyCandidate.a.id, columnB: keyCandidate.b.id, confidence: round(keyCandidate.score), @@ -23,6 +27,7 @@ export function suggestColumnMappings(fileA, fileB, dataA, dataB) { for (const columnB of columnsB) { if (usedB.has(columnB.id)) continue; if (!compatibleTypes(columnA.inferredType, columnB.inferredType)) continue; + if (!semanticallyCompatible(columnA.label, columnB.label)) continue; const score = comparePairScore(columnA, columnB, dataA.rows, dataB.rows); if (!best || score > best.score) best = { a: columnA, b: columnB, score }; @@ -30,6 +35,7 @@ export function suggestColumnMappings(fileA, fileB, dataA, dataB) { if (best && best.score >= 0.48) { suggestions.push({ + role: 'comparison', columnA: best.a.id, columnB: best.b.id, confidence: round(best.score), @@ -42,11 +48,12 @@ export function suggestColumnMappings(fileA, fileB, dataA, dataB) { } function bestKeyPair(columnsA, columnsB, rowsA, rowsB) { - let best = null; + const candidates = []; for (const a of columnsA) { for (const b of columnsB) { if (!compatibleTypes(a.inferredType, b.inferredType)) continue; + if (!semanticallyCompatible(a.label, b.label)) continue; const valuesA = nonEmpty(rowsA.map((row) => row[a.id])); const valuesB = nonEmpty(rowsB.map((row) => row[b.id])); @@ -55,14 +62,36 @@ function bestKeyPair(columnsA, columnsB, rowsA, rowsB) { const overlap = overlapRatio(valuesA, valuesB); const uniqueness = Math.min(uniqueRatio(valuesA), uniqueRatio(valuesB)); const semantic = semanticSimilarity(a.label, b.label); - const score = overlap * 0.62 + uniqueness * 0.25 + semantic * 0.13; - if (!best || score > best.score) best = { a, b, score }; + candidates.push({ + a, + b, + score, + identifier: semanticGroup(a.label) === 'identifier' && + semanticGroup(b.label) === 'identifier', + }); } } - return best && best.score >= 0.35 ? best : null; + if (!candidates.length) return null; + + const identifierCandidates = candidates.filter((candidate) => ( + candidate.identifier && candidate.score >= 0.35 + )); + + const pool = (identifierCandidates.length ? identifierCandidates : candidates) + .sort((left, right) => right.score - left.score); + + const best = pool[0]; + if (!best || best.score < MIN_KEY_SCORE) return null; + + const runnerUp = pool[1]; + if (runnerUp && best.score - runnerUp.score < MIN_KEY_MARGIN) { + return null; + } + + return best; } function comparePairScore(a, b, rowsA, rowsB) { @@ -81,6 +110,14 @@ function compatibleTypes(a, b) { return a === b; } +function semanticallyCompatible(a, b) { + const groupA = semanticGroup(a); + const groupB = semanticGroup(b); + + if (!groupA || !groupB) return true; + return groupA === groupB; +} + function semanticSimilarity(a, b) { const groupA = semanticGroup(a); const groupB = semanticGroup(b); diff --git a/src/models/contracts.js b/src/models/contracts.js index 313c721..00ecd86 100644 --- a/src/models/contracts.js +++ b/src/models/contracts.js @@ -41,6 +41,14 @@ * @typedef {'text' | 'number' | 'date'} MappingFieldType */ +/** + * @typedef {Object} MappingSuggestion + * @property {'key' | 'comparison'} role + * @property {string} columnA + * @property {string} columnB + * @property {number} confidence + */ + /** * @typedef {Object} Tolerance * @property {'absolute' | 'percentage'} mode diff --git a/src/services/mock-comparison-service.js b/src/services/mock-comparison-service.js index db585a1..826300f 100644 --- a/src/services/mock-comparison-service.js +++ b/src/services/mock-comparison-service.js @@ -81,7 +81,12 @@ export class MockComparisonService extends ComparisonService { fileA.columns.some((c) => c.id === a) && fileB.columns.some((c) => c.id === b) ) { - suggestions.push({ columnA: a, columnB: b, confidence: 0.91 }); + suggestions.push({ + role: suggestions.length === 0 ? 'key' : 'comparison', + columnA: a, + columnB: b, + confidence: 0.91, + }); } } return suggestions; diff --git a/tests/browser-comparison-service.test.js b/tests/browser-comparison-service.test.js index d697ac5..73fe4a4 100644 --- a/tests/browser-comparison-service.test.js +++ b/tests/browser-comparison-service.test.js @@ -33,18 +33,33 @@ test('real browser service inspects files, suggests mappings and compares actual const suggestions = await service.suggestMappings(fileA, fileB); + assert.equal(suggestions[0].role, 'key'); assert.equal(suggestions[0].columnA, 'Reference'); assert.equal(suggestions[0].columnB, 'Payment Ref'); - assert.ok(suggestions.some((item) => item.columnA === 'Amount' && item.columnB === 'Total')); - assert.ok(suggestions.some((item) => item.columnA === 'Date' && item.columnB === 'Paid Date')); + assert.ok(suggestions.some((item) => ( + item.role === 'comparison' && + item.columnA === 'Amount' && + item.columnB === 'Total' + ))); + assert.ok(suggestions.some((item) => ( + item.role === 'comparison' && + item.columnA === 'Date' && + item.columnB === 'Paid Date' + ))); + assert.ok(!suggestions.some((item) => ( + item.columnA === 'Description' && + item.columnB === 'Customer' + ))); - const key = suggestions[0]; - const comparisons = suggestions.slice(1).map((item) => ({ - columnA: item.columnA, - columnB: item.columnB, - type: fileA.columns.find((column) => column.id === item.columnA)?.inferredType ?? 'text', - tolerance: { mode: 'absolute', value: 0.01 }, - })); + const key = suggestions.find((item) => item.role === 'key'); + const comparisons = suggestions + .filter((item) => item.role === 'comparison') + .map((item) => ({ + columnA: item.columnA, + columnB: item.columnB, + type: fileA.columns.find((column) => column.id === item.columnA)?.inferredType ?? 'text', + tolerance: { mode: 'absolute', value: 0.01 }, + })); const result = await service.compare({ fileAId: fileA.id, @@ -77,3 +92,60 @@ test('real browser service inspects files, suggests mappings and compares actual const inv1011 = result.records.find((record) => record.keyLabel === 'inv-1011'); assert.equal(inv1011.status, 'MISMATCH'); }); + +test('does not suggest semantically conflicting text columns even with perfect value overlap', async () => { + const service = new BrowserComparisonService(); + + const fileA = await service.inspectFile(fakeFile( + 'a.csv', + 'Reference,Description\nINV-1,Acme\nINV-2,Beta\n' + ), { slot: 'A' }); + + const fileB = await service.inspectFile(fakeFile( + 'b.csv', + 'Payment Ref,Customer\nINV-1,Acme\nINV-2,Beta\n' + ), { slot: 'B' }); + + const suggestions = await service.suggestMappings(fileA, fileB); + + assert.equal(suggestions[0].role, 'key'); + assert.ok(!suggestions.some((item) => ( + item.columnA === 'Description' && + item.columnB === 'Customer' + ))); +}); + +test('refuses automatic mapping when two identifier keys are equally plausible', async () => { + const service = new BrowserComparisonService(); + + const text = [ + 'Primary ID,Secondary ID,Amount', + 'A1,X1,10.00', + 'A2,X2,20.00', + 'A3,X3,30.00', + '', + ].join('\n'); + + const fileA = await service.inspectFile(fakeFile('a.csv', text), { slot: 'A' }); + const fileB = await service.inspectFile(fakeFile('b.csv', text), { slot: 'B' }); + + const suggestions = await service.suggestMappings(fileA, fileB); + + assert.deepEqual(suggestions, []); +}); + +test('CSV export protects formula-like key cells', async () => { + const service = new BrowserComparisonService(); + + const blob = await service.createExport({ + records: [{ + status: 'ONLY_A', + keyLabel: '=2+2', + source: { rowA: 2, rowB: null }, + differences: [], + }], + }); + + const csv = await blob.text(); + assert.match(csv, /ONLY_A,'=2\+2,2,,/); +}); diff --git a/tests/delimited-parser.test.js b/tests/delimited-parser.test.js index 270e082..993b45a 100644 --- a/tests/delimited-parser.test.js +++ b/tests/delimited-parser.test.js @@ -12,12 +12,20 @@ test('detects semicolon delimiter', () => { assert.equal(detectDelimiter(text), ';'); }); -test('parses quoted commas and multiline values', () => { - const text = 'id,note\n1,"hello, world"\n2,"line one\nline two"\n'; +test('detects and parses TSV input', () => { + const table = tableFromText('id\tamount\nA1\t12.50\nA2\t13.00\n'); + + assert.equal(table.delimiter, '\t'); + assert.equal(table.rows[1].amount, '13.00'); +}); + +test('parses quoted commas, escaped quotes and multiline values', () => { + const text = 'id,note\n1,"hello, world"\n2,"say ""hello"""\n3,"line one\nline two"\n'; const rows = parseDelimitedText(text, ','); assert.deepEqual(rows[1], ['1', 'hello, world']); - assert.deepEqual(rows[2], ['2', 'line one\nline two']); + assert.deepEqual(rows[2], ['2', 'say "hello"']); + assert.deepEqual(rows[3], ['3', 'line one\nline two']); }); test('strips UTF-8 BOM and infers basic types', () => { @@ -27,3 +35,31 @@ test('strips UTF-8 BOM and infers basic types', () => { assert.equal(table.columns[2].inferredType, 'number'); assert.equal(table.rows[0].id, 'A1'); }); + +test('rejects malformed unclosed quoted fields', () => { + assert.throws( + () => tableFromText('id,note\n1,"unfinished\n'), + /CSV_PARSE_UNCLOSED_QUOTE/ + ); +}); + +test('rejects replacement characters instead of accepting corrupted UTF-8', () => { + assert.throws( + () => tableFromText('id,name\n1,broken\uFFFDvalue\n'), + /UNSUPPORTED_ENCODING/ + ); +}); + +test('deduplicates repeated headers deterministically', () => { + const table = tableFromText('Name,Name\nAlpha,Beta\n'); + + assert.deepEqual( + table.columns.map((column) => column.id), + ['Name', 'Name (2)'] + ); + assert.equal(table.rows[0]['Name (2)'], 'Beta'); +}); + +test('rejects empty files', () => { + assert.throws(() => tableFromText(''), /EMPTY_FILE/); +}); diff --git a/tests/normalization.test.js b/tests/normalization.test.js new file mode 100644 index 0000000..a8df93a --- /dev/null +++ b/tests/normalization.test.js @@ -0,0 +1,54 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { parseFlexibleNumber } from '../src/engine/delimited-parser.js'; +import { + compareValues, + normalizeKeyForSide, + normalizeText, +} from '../src/engine/normalization.js'; + +test('parses simple dot and comma decimal values', () => { + assert.equal(parseFlexibleNumber('12.50'), 12.5); + assert.equal(parseFlexibleNumber('12,50'), 12.5); +}); + +test('normalizes key text consistently across sides', () => { + const mappings = [{ columnA: 'Reference', columnB: 'Payment Ref' }]; + const config = { + trimText: true, + caseInsensitive: true, + collapseWhitespace: true, + }; + + const keyA = normalizeKeyForSide( + { Reference: ' INV 1001 ' }, + mappings, + 'A', + config + ); + const keyB = normalizeKeyForSide( + { 'Payment Ref': 'inv 1001' }, + mappings, + 'B', + config + ); + + assert.equal(keyA, keyB); + assert.equal(normalizeText(' A B ', config), 'a b'); +}); + +test('applies percentage tolerance deterministically', () => { + const inside = compareValues('100', '101', { + type: 'number', + tolerance: { mode: 'percentage', value: 1 }, + }); + + const outside = compareValues('100', '102', { + type: 'number', + tolerance: { mode: 'percentage', value: 1 }, + }); + + assert.equal(inside.equal, true); + assert.equal(outside.equal, false); +});