diff --git a/README.md b/README.md index c6eee00..6d5e667 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ const config = parse(text) config.scope = 'local' config.database.database = 'use_another_database' +config.database.description = 'this is a multiline\n value -- continuation lines\n must be indented' config.paths.default.tmpdir = '/tmp' delete config.paths.default.datadir config.paths.default.array.push('fourth value') @@ -71,6 +72,9 @@ scope=local user=dbuser password=dbpassword database=use_another_database +description=this is a multiline + value -- continuation lines + must be indented [section.paths.default] tmpdir=/tmp array[]=first value @@ -87,7 +91,23 @@ Attempts to turn the given INI string into a nested data object. ```js // You can also use `decode` -const object = parse(``) +const object = parse(``, { + /** + * Read indented lines that directly follow a key/value pair as + * continuation lines of that value, the way Python's configparser + * does. Enabled by default; see "Multiline values" below. + * + * Set to `false` to read those indented lines as standalone keys, + * which is how versions before multiline support read them. + */ + multiline: true, + + /** + * Whether to treat repeated keys without a `[]` suffix as arrays. + * Enabled by default to match the .ini format used by npm. + */ + bracketedArray: true, +}) ``` ### Stringify @@ -152,7 +172,19 @@ stringify(object,{ * Some parsers treat duplicate names by themselves as arrays */ - bracketedArray : true + bracketedArray : true, + + /** + * A string value containing newlines is written as indented + * continuation lines when `parse()` can read it back unchanged + * (see "Multiline values" below), and JSON-quoted otherwise, + * as before multiline support. + * + * Set to `true` to have `stringify()` throw instead of falling + * back to JSON quoting, for output that must stay readable by + * parsers that do not understand quoted values. + */ + strictMultiline : false }) ``` @@ -164,6 +196,25 @@ stringify(object,{ stringify(object,'section') ``` +### Multiline values + +A value continues onto the following lines when each of them starts with +a space or tab. The continuation lines are kept verbatim, indentation +included, joined with `\n`: + +```ini +description=first line + second line + third line +``` + +`parse()` ends the value at a blank line, a section header, or any line +that is not indented, and skips comment lines in between. `stringify()` +writes a value this way only when every line after the first is indented, +is not blank, contains no `=`, and does not start with `;` or `#`, and the +value contains no carriage return. Any other value with newlines is +JSON-quoted, so `parse(stringify(x))` gives back `x` either way. + ### Un / Escape Turn the given string into a safe to diff --git a/lib/ini.js b/lib/ini.js index beb390d..5bcf2e4 100644 --- a/lib/ini.js +++ b/lib/ini.js @@ -1,5 +1,29 @@ const { hasOwnProperty } = Object.prototype +// A string value with embedded newlines can be written verbatim as indented +// continuation lines only if the decoder will read every one of those lines +// back as part of the same value. Returns the lines to write, or null when +// the value has to be JSON-quoted instead. +const continuationLines = val => { + // The decoder drops a CR that precedes LF, and on win32 the platform eol + // would add a second one. + if (val.includes('\r')) { + return null + } + const lines = val.split('\n') + for (const ln of lines.slice(1)) { + if ( + !/^[ \t]/.test(ln) || // must be indented + !/\S/.test(ln) || // a blank line ends the value + ln.includes('=') || // would parse as a new key + /^\s*[;#]/.test(ln) // would be skipped as a comment + ) { + return null + } + } + return lines +} + const encode = (obj, opt = {}) => { if (typeof opt === 'string') { opt = { section: opt } @@ -8,6 +32,7 @@ const encode = (obj, opt = {}) => { opt.newline = opt.newline === true opt.sort = opt.sort === true opt.whitespace = opt.whitespace === true || opt.align === true + opt.strictMultiline = opt.strictMultiline === true // The `typeof` check is required because accessing the `process` directly fails on browsers. /* istanbul ignore next */ opt.platform = opt.platform || (typeof process !== 'undefined' && process.platform) @@ -43,16 +68,36 @@ const encode = (obj, opt = {}) => { let out = '' const arraySuffix = opt.bracketedArray ? '[]' : '' + // Write one key/value line, as indented continuation lines when the value + // allows it. + const entry = (key, val) => { + let text = null + if (typeof val === 'string' && val.includes('\n')) { + const lines = continuationLines(val) + if (lines) { + // The first line is escaped like any other value; the continuation + // lines are read back verbatim. + text = safe(lines[0]) + eol + lines.slice(1).join(eol) + } else if (opt.strictMultiline) { + throw new Error(`Value of "${key}" has an embedded newline but cannot be written as indented continuation lines. Disable strictMultiline to allow such a value.`) + } + } + if (text === null) { + text = safe(val) + } + out += safe(key).padEnd(padToChars, ' ') + separator + text + eol + } + for (const k of keys) { const val = obj[k] if (val && Array.isArray(val)) { for (const item of val) { - out += safe(`${k}${arraySuffix}`).padEnd(padToChars, ' ') + separator + safe(item) + eol + entry(`${k}${arraySuffix}`, item) } } else if (val && typeof val === 'object') { children.push(k) } else { - out += safe(k).padEnd(padToChars, ' ') + separator + safe(val) + eol + entry(k, val) } } @@ -105,18 +150,35 @@ function splitSections (str, separator) { const decode = (str, opt = {}) => { opt.bracketedArray = opt.bracketedArray !== false + opt.multiline = opt.multiline !== false const out = Object.create(null) let p = out let section = null // section |key = value const re = /^\[([^\]]*)\]\s*$|^([^=]+)(=(.*))?$/i - const lines = str.split(/[\r\n]+/g) + const lines = str.split(/\r?\n/g) const duplicates = {} + // While an indented line may still continue the most recent string value, + // `cont` says where that value lives: [object or array, key or index]. + // Anything other than a comment or a continuation line clears it. + let cont = null for (const line of lines) { - if (!line || line.match(/^\s*[;#]/) || line.match(/^\s*$/)) { + if (line.match(/^\s*[;#]/)) { + // comments do not break continuation, just skip + continue + } + if (line.match(/^\s*$/)) { + // blank lines break continuation + cont = null continue } + if (cont && /^[ \t]/.test(line) && line.indexOf('=') === -1) { + const [target, slot] = cont + target[slot] += '\n' + line + continue + } + cont = null const match = line.match(re) if (!match) { continue @@ -165,8 +227,14 @@ const decode = (str, opt = {}) => { // array by accidentally forgetting the brackets if (Array.isArray(p[key])) { p[key].push(value) + if (opt.multiline && typeof value === 'string') { + cont = [p[key], p[key].length - 1] + } } else { p[key] = value + if (opt.multiline && typeof value === 'string') { + cont = [p, key] + } } } diff --git a/tap-snapshots/test/foo.js.test.cjs b/tap-snapshots/test/foo.js.test.cjs index 2646323..6fc4f89 100644 --- a/tap-snapshots/test/foo.js.test.cjs +++ b/tap-snapshots/test/foo.js.test.cjs @@ -39,6 +39,176 @@ Null Object { "br": "warm", "eq": "eq=eq", "false": false, + "multiline": Null Object { + "alpha": Null Object { + "a1": "one", + "a2": "two", + "a3": "three", + "b1": "two", + "five": String( + line1 + line2 + \\tline3 + line4 + \\t line5 + ), + "two": String( + first + second + ), + }, + "array": Null Object { + "list": Array [ + "item1", + String( + line1 + line2 + \\tline3 + ), + String( + ok + good + ), + String( + good + indented + ), + ], + }, + "beta": Null Object { + "b1": "after", + "mstart": String( + top + middle + \\tbottom + ), + }, + "delta": Null Object { + "d1": "whitespace on following newline", + "d2": "done", + "dml": "first", + "second": true, + "third": true, + }, + "gamma": Null Object { + "g1": "before", + "mnext": String( + x + y + \\tz + ), + }, + }, + "null": null, + "o": "p", + "s": "something", + "s1": "\\"something'", + "s2": "something else", + "s3": "", + "s4": "", + "s5": " ", + "s6": " a ", + "s7": true, + "true": true, + "undefined": "undefined", + "x.y.z": Null Object { + "a.b.c": Null Object { + "a.b.c": "abc", + "nocomment": "this; this is not a comment", + "noHashComment": "this# this is not a comment", + }, + "x.y.z": "xyz", + }, + "zr": Array [ + "deedee", + ], +} +` + +exports[`test/foo.js TAP decode from file with multiline disabled > must match snapshot 1`] = ` +Null Object { + " xa n p ": String( + "\\r + yoyoyo\\r\\r + + ), + "[disturbing]": "hey you never know", + "a": Null Object { + "[]": "a square?", + "av": "a val", + "b": Null Object { + "c": Null Object { + "e": "1", + "j": "2", + }, + }, + "cr": Array [ + "four", + "eight", + ], + "e": "{ o: p, a: { av: a val, b: { c: { e: \\"this [value]\\" } } } }", + "j": "\\"{ o: \\"p\\", a: { av: \\"a val\\", b: { c: { e: \\"this [value]\\" } } } }\\"", + }, + "a with spaces": "b c", + "ar": Array [ + "one", + "three", + "this is included", + ], + "b": Null Object {}, + "br": "warm", + "eq": "eq=eq", + "false": false, + "multiline": Null Object { + "alpha": Null Object { + "a1": "one", + "a2": "two", + "a3": "three", + "b1": "two", + "five": "line1", + "line2": true, + "line3": true, + "line4": true, + "line5": true, + "multiline value across": true, + "second": true, + "three lines": true, + "two": "first", + }, + "array": Null Object { + "good": true, + "line2": true, + "line3": true, + "list": Array [ + "item1", + "line1", + "ok", + String( + good + indented + ), + ], + }, + "beta": Null Object { + "b1": "after", + "bottom": true, + "middle": true, + "mstart": "top", + }, + "delta": Null Object { + "d1": "whitespace on following newline", + "d2": "done", + "dml": "first", + "second": true, + "third": true, + }, + "gamma": Null Object { + "g1": "before", + "mnext": "x", + "y": true, + "z": true, + }, + }, "null": null, "o": "p", "s": "something", @@ -109,6 +279,48 @@ a.b.c=abc nocomment=this\\; this is not a comment noHashComment=this\\# this is not a comment +[multiline.alpha] +a1=one +a2=two +b1=two +two=first + second +five=line1 + line2 + line3 + line4 + line5 +a3=three + +[multiline.beta] +mstart=top + middle + bottom +b1=after + +[multiline.gamma] +g1=before +mnext=x + y + z + +[multiline.delta] +dml=first +second=true +third=true +d1=whitespace on following newline +d2=done + +[multiline.array] +list[]=item1 +list[]=line1 + line2 + line3 +list[]=ok + good +list[]=good + indented + ` exports[`test/foo.js TAP encode with align > must match snapshot 1`] = ` @@ -155,6 +367,48 @@ a.b.c = abc nocomment = this\\; this is not a comment noHashComment = this\\# this is not a comment +[multiline.alpha] +a1 = one +a2 = two +b1 = two +two = first + second +five = line1 + line2 + line3 + line4 + line5 +a3 = three + +[multiline.beta] +mstart = top + middle + bottom +b1 = after + +[multiline.gamma] +g1 = before +mnext = x + y + z + +[multiline.delta] +dml = first +second = true +third = true +d1 = whitespace on following newline +d2 = done + +[multiline.array] +list[] = item1 +list[] = line1 + line2 + line3 +list[] = ok + good +list[] = good + indented + ` exports[`test/foo.js TAP encode with align and sort > must match snapshot 1`] = ` @@ -193,6 +447,48 @@ j = "\\"{ o: \\"p\\", a: { av: \\"a val\\", b: { c: { e: \\"this [value]\\" } e = 1 j = 2 +[multiline.alpha] +a1 = one +a2 = two +a3 = three +b1 = two +five = line1 + line2 + line3 + line4 + line5 +two = first + second + +[multiline.array] +list[] = item1 +list[] = line1 + line2 + line3 +list[] = ok + good +list[] = good + indented + +[multiline.beta] +b1 = after +mstart = top + middle + bottom + +[multiline.delta] +d1 = whitespace on following newline +d2 = done +dml = first +second = true +third = true + +[multiline.gamma] +g1 = before +mnext = x + y + z + [x\\.y\\.z] x.y.z = xyz @@ -273,6 +569,48 @@ j="\\"{ o: \\"p\\", a: { av: \\"a val\\", b: { c: { e: \\"this [value]\\" } } } e=1 j=2 +[multiline.alpha] +a1=one +a2=two +a3=three +b1=two +five=line1 + line2 + line3 + line4 + line5 +two=first + second + +[multiline.array] +list[]=item1 +list[]=line1 + line2 + line3 +list[]=ok + good +list[]=good + indented + +[multiline.beta] +b1=after +mstart=top + middle + bottom + +[multiline.delta] +d1=whitespace on following newline +d2=done +dml=first +second=true +third=true + +[multiline.gamma] +g1=before +mnext=x + y + z + [x\\.y\\.z] x.y.z=xyz @@ -302,3 +640,10 @@ label=debug value=10 ` + +exports[`test/foo.js TAP legacy encode preserves problematic multiline entry > must match snapshot 1`] = ` +Array [ + "\\" xa n p \\"=\\"\\\\\\"\\\\r\\\\nyoyoyo\\\\r\\\\r\\\\n\\"", + "", +] +` diff --git a/test/fixtures/foo-multiline-error.ini b/test/fixtures/foo-multiline-error.ini new file mode 100644 index 0000000..80e462a --- /dev/null +++ b/test/fixtures/foo-multiline-error.ini @@ -0,0 +1,6 @@ +; The same entry as in foo.ini: a value with embedded newlines that cannot +; be written as indented continuation lines, so encode() JSON-quotes it and +; strictMultiline rejects it. + +; wrap in quotes to JSON-decode and preserve spaces +" xa n p " = "\"\r\nyoyoyo\r\r\n" diff --git a/test/fixtures/foo.ini b/test/fixtures/foo.ini index 219b4aa..f3f50d5 100644 --- a/test/fixtures/foo.ini +++ b/test/fixtures/foo.ini @@ -93,3 +93,56 @@ nocomment = this\; this is not a comment # this next one is not a comment! it's escaped! noHashComment = this\# this is not a comment + +; Multiline entry testing + +[multiline.alpha] +a1=one +a2 = a typical long + multiline value across + three lines +b1=two +two = first + second + a2 = two +five=line1 + line2 +# tab here: + line3 + line4 +;tab here: + line5 +a3=three + +[multiline.beta] +mstart=top + middle + bottom +b1=after + +[multiline.gamma] +g1=before +mnext=x + y + ; tab here: + z + +[multiline.delta] + +dml=first + + second + third + +d1=whitespace on following newline + +d2=done + +[multiline.array] +list[]=item1 +list[]=line1 + line2 + line3 +list[]=ok + good +list[]="good\n indented" diff --git a/test/foo.js b/test/foo.js index fe92435..a6bcd90 100644 --- a/test/foo.js +++ b/test/foo.js @@ -5,6 +5,8 @@ const fs = require('fs') const path = require('path') const fixture = path.resolve(__dirname, './fixtures/foo.ini') const data = fs.readFileSync(fixture, 'utf8') +const errorFixture = path.resolve(__dirname, './fixtures/foo-multiline-error.ini') +const errorData = fs.readFileSync(errorFixture, 'utf8') tap.cleanSnapshot = s => s.replace(/\r\n/g, '\n') @@ -14,6 +16,12 @@ test('decode from file', function (t) { t.end() }) +test('decode from file with multiline disabled', function (t) { + const d = i.decode(data, { multiline: false }) + t.matchSnapshot(d) + t.end() +}) + test('encode from data', function (t) { const d = i.decode(data) const e = i.encode(d) @@ -94,3 +102,77 @@ test('encode within browser context', function (t) { t.matchSnapshot(e) t.end() }) + +test('legacy encode preserves problematic multiline entry', function (t) { + const obj = i.decode(errorData) + const encoded = i.encode(obj, { strictMultiline: false }) + // make sure the lone \r is treated as newline on Windows + t.matchSnapshot(encoded.split(/\r?\n/)) + t.end() +}) + +test('strict encode fails on problematic multiline entry', function (t) { + const obj = i.decode(errorData) + t.throws(() => i.encode(obj, { strictMultiline: true })) + t.end() +}) + +// Every value here is one the verbatim continuation form cannot carry, so +// encode() falls back to JSON quoting: parse(stringify(x)) must still be x, +// and strictMultiline must reject the value instead. +const quoted = { + 'unindented line': ['line1\nline2', '"line1\\nline2"'], + 'carriage return': ['a\r\n b', '"a\\r\\n b"'], + 'blank line inside the value': ['x\n\n y', '"x\\n\\n y"'], + 'whitespace-only line': ['x\n \n y', '"x\\n \\n y"'], + 'trailing newline': ['x\n y\n', '"x\\n y\\n"'], + 'equals sign in a continuation line': ['x\n y=z', '"x\\n y=z"'], + 'comment-looking continuation line': ['x\n ;y', '"x\\n ;y"'], +} + +for (const [name, [value, expected]] of Object.entries(quoted)) { + test(`quoted fallback: ${name}`, function (t) { + const obj = { k: value, list: [value] } + const e = i.encode(obj) + t.same(e.split(/\r?\n/), [`k=${expected}`, `list[]=${expected}`, '']) + t.same(i.decode(e), obj, 'round trip') + t.throws(() => i.encode(obj, { strictMultiline: true }), /continuation lines/) + t.end() + }) +} + +// Values the verbatim form can carry, including a first line that needs +// escaping and array entries. +test('continuation lines round trip', function (t) { + const obj = { + semi: 'x;y\n z', + quoted: '"q"\n y', + empty: '\n x', + list: ['a\n b', 'c\n\td'], + } + const e = i.encode(obj) + t.same(e.split(/\r?\n/), [ + 'semi=x\\;y', ' z', + 'quoted="\\"q\\""', ' y', + 'empty=', ' x', + 'list[]=a', ' b', + 'list[]=c', '\td', + '', + ]) + t.same(i.decode(e), obj, 'round trip') + t.end() +}) + +test('decode: what ends a continuation', function (t) { + t.same(i.decode('a=x\n y\n\n z'), { a: 'x\n y', z: true }, 'blank line') + t.same(i.decode('a=x\n y\n[s]\n z'), { a: 'x\n y', s: { z: true } }, 'section header') + t.same(i.decode('a=x\n y\n=junk\n z'), { a: 'x\n y', z: true }, 'unparseable line') + t.same(i.decode('a=x\n ; c\n y'), { a: 'x\n y' }, 'a comment does not') + t.end() +}) + +test('decode: array entries continue', function (t) { + t.same(i.decode('a[]=x\n y\na[]=z\n w'), { a: ['x\n y', 'z\n w'] }) + t.same(i.decode('a=x\n y\na=z\n w', { bracketedArray: false }), { a: ['x\n y', 'z\n w'] }) + t.end() +})