diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97ebcb9..2b6763e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: run: npm test cross-test-python: - name: Cross-test vs Python (PyPI 0.2.2) + name: Cross-test vs Python (PyPI 0.2.3) runs-on: ubuntu-latest steps: @@ -41,18 +41,18 @@ jobs: with: python-version: '3.11' - - name: Install Python NSV 0.2.2 - run: pip install nsv==0.2.2 + - name: Install Python NSV 0.2.3 + run: pip install nsv==0.2.3 - name: Run cross-tests against Python run: node cross-test-python.js cross-test-rust: - name: Cross-test vs Rust (crates.io 0.0.9) + name: Cross-test vs Rust (crates.io 0.0.12) runs-on: ubuntu-latest env: - NSV_RUST_VERSION: "0.0.9" + NSV_RUST_VERSION: "0.0.12" steps: - uses: actions/checkout@v4 diff --git a/cross-test-python.js b/cross-test-python.js index f9d13fd..5d7319b 100644 --- a/cross-test-python.js +++ b/cross-test-python.js @@ -20,7 +20,7 @@ const tests = [ { name: 'complex example', input: 'first\nrow\n\nsecond\nrow\n\nmissing ->\n\\\n<- missing\n\nRoses are red\\nViolets are blue\\nThis may be pain\\nBut CSV would be, too\nTab\\tseparated\\tvalues\\n(would be left as-is normally)\nNot a newline: \\\\n\n' }, ]; -console.log('Cross-testing JS implementation against Python (PyPI 0.2.2)\n'); +console.log('Cross-testing JS implementation against Python (PyPI 0.2.3)\n'); console.log('='.repeat(60) + '\n'); let passCount = 0; diff --git a/cross-test-rust.js b/cross-test-rust.js index dd106a7..babf4ae 100644 --- a/cross-test-rust.js +++ b/cross-test-rust.js @@ -3,8 +3,8 @@ const { execSync } = require('child_process'); const fs = require('fs'); const path = require('path'); -// Use NSV_RUST_VERSION env var if set, otherwise default to 0.0.9 -const nsvVersion = process.env.NSV_RUST_VERSION || '0.0.9'; +// Use NSV_RUST_VERSION env var if set, otherwise default to 0.0.12 +const nsvVersion = process.env.NSV_RUST_VERSION || '0.0.12'; // Test cases const tests = [ diff --git a/index.js b/index.js index 38e7f4e..5986184 100644 --- a/index.js +++ b/index.js @@ -145,7 +145,7 @@ async function read(input) { input.on('end', () => { // Handle both Buffer and string chunks const text = chunks.map(chunk => - typeof chunk === 'string' ? chunk : chunk.toString('utf8') + typeof chunk === 'string' ? chunk : chunk.toString('latin1') ).join(''); try { resolve(parse(text)); @@ -259,7 +259,7 @@ class Reader { // Otherwise set up stream handlers this.input.on('data', (chunk) => { try { - const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8'); + const text = typeof chunk === 'string' ? chunk : chunk.toString('latin1'); this._processChunk(text); } catch (error) { this._error = error; diff --git a/test-streaming.js b/test-streaming.js index 55be08c..310940f 100644 --- a/test-streaming.js +++ b/test-streaming.js @@ -160,12 +160,92 @@ async function testEmptyRows() { console.log('✓ Empty rows handled correctly\n'); } +// Test 5: Buffer chunks are bytes — result independent of chunk boundaries +async function testBufferChunksAreBytes() { + console.log('Test 5: Buffer chunks treated as bytes, invariant under chunk boundaries'); + + function bufferStream(chunks) { + let index = 0; + return new Readable({ + read() { + this.push(index < chunks.length ? chunks[index++] : null); + } + }); + } + + async function readBoth(chunks) { + const readerRows = []; + for await (const row of new nsv.Reader(bufferStream(chunks))) { + readerRows.push(row); + } + const readRows = await nsv.read(bufferStream(chunks)); + return [['Reader', readerRows], ['read()', readRows]]; + } + + const cases = [ + { name: '2-byte (é)', text: 'café\n\né\n\n' }, + { name: '3-byte (★)', text: 'a★b\n\n★\n\n' }, + { name: '4-byte (🎉)', text: 'a\u{1f389}b\n\n\u{1f389}\n\n' }, + ]; + + for (const { name, text } of cases) { + const bytes = Buffer.from(text, 'utf8'); + const expected = nsv.parse(bytes.toString('latin1')); + + const recovered = expected.map(row => + row.map(cell => Buffer.from(cell, 'latin1').toString('utf8'))); + if (JSON.stringify(recovered) !== JSON.stringify(nsv.parse(text))) { + console.error(`✗ ${name}: latin1 transport is not byte-faithful`); + console.error(' Expected:', nsv.parse(text)); + console.error(' Got:', recovered); + process.exit(1); + } + + for (let split = 0; split <= bytes.length; split++) { + const chunks = [bytes.slice(0, split), bytes.slice(split)]; + for (const [path, rows] of await readBoth(chunks)) { + if (JSON.stringify(rows) !== JSON.stringify(expected)) { + console.error(`✗ ${name} via ${path}, split at byte ${split}`); + console.error(' Expected:', expected); + console.error(' Got:', rows); + process.exit(1); + } + } + } + console.log(` ✓ ${name}: byte-faithful, all ${bytes.length + 1} split positions, Reader and read()`); + } + + { + const text = 'a\\nb\n\né\u{1f389}\n\n'; + const bytes = Buffer.from(text, 'utf8'); + const expected = nsv.parse(bytes.toString('latin1')); + const escapeSplit = 2; + const codePointSplit = bytes.length - 4; + const chunks = [ + bytes.slice(0, escapeSplit), + bytes.slice(escapeSplit, codePointSplit), + bytes.slice(codePointSplit), + ]; + for (const [path, rows] of await readBoth(chunks)) { + if (JSON.stringify(rows) !== JSON.stringify(expected)) { + console.error(`✗ combined escape+code-point split via ${path}`); + console.error(' Expected:', expected); + console.error(' Got:', rows); + process.exit(1); + } + } + console.log(' ✓ combined: boundary in escape sequence + boundary in code point'); + } + console.log('✓ Buffer chunks handled as bytes correctly\n'); +} + // Run all tests (async () => { await testChunkedReading(); await testIncrementalWriting(); await testInfiniteStream(); await testEmptyRows(); + await testBufferChunksAreBytes(); console.log('✓ All streaming tests passed!'); })().catch(error => { console.error('✗ Test failed:', error);