latin1

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
       run: npm test
 
   cross-test-python:
-    name: Cross-test vs Python (PyPI 0.2.2)
+    name: Cross-test vs Python (PyPI 0.2.3)
     runs-on: ubuntu-latest
 
     steps:
@@ -41,18 +41,18 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Install Python NSV 0.2.2
-      run: pip install nsv==0.2.2
+    - name: Install Python NSV 0.2.3
+      run: pip install nsv==0.2.3
 
     - name: Run cross-tests against Python
       run: node cross-test-python.js
 
   cross-test-rust:
-    name: Cross-test vs Rust (crates.io 0.0.9)
+    name: Cross-test vs Rust (crates.io 0.0.12)
     runs-on: ubuntu-latest
 
     env:
-      NSV_RUST_VERSION: "0.0.9"
+      NSV_RUST_VERSION: "0.0.12"
 
     steps:
     - uses: actions/checkout@v4

cross-test-python.js

@@ -20,7 +20,7 @@ const tests = [
   { name: 'complex example', input: 'first\nrow\n\nsecond\nrow\n\nmissing ->\n\\\n<- missing\n\nRoses are red\\nViolets are blue\\nThis may be pain\\nBut CSV would be, too\nTab\\tseparated\\tvalues\\n(would be left as-is normally)\nNot a newline: \\\\n\n' },
 ];
 
-console.log('Cross-testing JS implementation against Python (PyPI 0.2.2)\n');
+console.log('Cross-testing JS implementation against Python (PyPI 0.2.3)\n');
 console.log('='.repeat(60) + '\n');
 
 let passCount = 0;

cross-test-rust.js

@@ -3,8 +3,8 @@ const { execSync } = require('child_process');
 const fs = require('fs');
 const path = require('path');
 
-// Use NSV_RUST_VERSION env var if set, otherwise default to 0.0.9
-const nsvVersion = process.env.NSV_RUST_VERSION || '0.0.9';
+// Use NSV_RUST_VERSION env var if set, otherwise default to 0.0.12
+const nsvVersion = process.env.NSV_RUST_VERSION || '0.0.12';
 
 // Test cases
 const tests = [

index.js

@@ -145,7 +145,7 @@ async function read(input) {
     input.on('end', () => {
       // Handle both Buffer and string chunks
       const text = chunks.map(chunk =>
-        typeof chunk === 'string' ? chunk : chunk.toString('utf8')
+        typeof chunk === 'string' ? chunk : chunk.toString('latin1')
       ).join('');
       try {
         resolve(parse(text));
@@ -259,7 +259,7 @@ class Reader {
     // Otherwise set up stream handlers
     this.input.on('data', (chunk) => {
       try {
-        const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
+        const text = typeof chunk === 'string' ? chunk : chunk.toString('latin1');
         this._processChunk(text);
       } catch (error) {
         this._error = error;

test-streaming.js

@@ -160,12 +160,92 @@ async function testEmptyRows() {
   console.log('✓ Empty rows handled correctly\n');
 }
 
+// Test 5: Buffer chunks are bytes — result independent of chunk boundaries
+async function testBufferChunksAreBytes() {
+  console.log('Test 5: Buffer chunks treated as bytes, invariant under chunk boundaries');
+
+  function bufferStream(chunks) {
+    let index = 0;
+    return new Readable({
+      read() {
+        this.push(index < chunks.length ? chunks[index++] : null);
+      }
+    });
+  }
+
+  async function readBoth(chunks) {
+    const readerRows = [];
+    for await (const row of new nsv.Reader(bufferStream(chunks))) {
+      readerRows.push(row);
+    }
+    const readRows = await nsv.read(bufferStream(chunks));
+    return [['Reader', readerRows], ['read()', readRows]];
+  }
+
+  const cases = [
+    { name: '2-byte (é)', text: 'café\n\né\n\n' },
+    { name: '3-byte (★)', text: 'a★b\n\n★\n\n' },
+    { name: '4-byte (🎉)', text: 'a\u{1f389}b\n\n\u{1f389}\n\n' },
+  ];
+
+  for (const { name, text } of cases) {
+    const bytes = Buffer.from(text, 'utf8');
+    const expected = nsv.parse(bytes.toString('latin1'));
+
+    const recovered = expected.map(row =>
+      row.map(cell => Buffer.from(cell, 'latin1').toString('utf8')));
+    if (JSON.stringify(recovered) !== JSON.stringify(nsv.parse(text))) {
+      console.error(`✗ ${name}: latin1 transport is not byte-faithful`);
+      console.error('  Expected:', nsv.parse(text));
+      console.error('  Got:', recovered);
+      process.exit(1);
+    }
+
+    for (let split = 0; split <= bytes.length; split++) {
+      const chunks = [bytes.slice(0, split), bytes.slice(split)];
+      for (const [path, rows] of await readBoth(chunks)) {
+        if (JSON.stringify(rows) !== JSON.stringify(expected)) {
+          console.error(`✗ ${name} via ${path}, split at byte ${split}`);
+          console.error('  Expected:', expected);
+          console.error('  Got:', rows);
+          process.exit(1);
+        }
+      }
+    }
+    console.log(`  ✓ ${name}: byte-faithful, all ${bytes.length + 1} split positions, Reader and read()`);
+  }
+
+  {
+    const text = 'a\\nb\n\né\u{1f389}\n\n';
+    const bytes = Buffer.from(text, 'utf8');
+    const expected = nsv.parse(bytes.toString('latin1'));
+    const escapeSplit = 2;
+    const codePointSplit = bytes.length - 4;
+    const chunks = [
+      bytes.slice(0, escapeSplit),
+      bytes.slice(escapeSplit, codePointSplit),
+      bytes.slice(codePointSplit),
+    ];
+    for (const [path, rows] of await readBoth(chunks)) {
+      if (JSON.stringify(rows) !== JSON.stringify(expected)) {
+        console.error(`✗ combined escape+code-point split via ${path}`);
+        console.error('  Expected:', expected);
+        console.error('  Got:', rows);
+        process.exit(1);
+      }
+    }
+    console.log('  ✓ combined: boundary in escape sequence + boundary in code point');
+  }
+  console.log('✓ Buffer chunks handled as bytes correctly\n');
+}
+
 // Run all tests
 (async () => {
   await testChunkedReading();
   await testIncrementalWriting();
   await testInfiniteStream();
   await testEmptyRows();
+  await testBufferChunksAreBytes();
   console.log('✓ All streaming tests passed!');
 })().catch(error => {
   console.error('✗ Test failed:', error);