Skip to content

fix(lexer): report precise reusable source ranges #721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeswr merged 2 commits into from
Sep 6, 2026
+311 −51
Merged

fix(lexer): report precise reusable source ranges #721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
27a5c0b
fix(lexer): report precise reusable source ranges
jeswr Sep 6, 2026
e99b7db
perf(lexer): simplify source range bookkeeping
jeswr Sep 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 90 additions & 27 deletions src/N3Lexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ export default class N3Lexer {
this._n3Verb = /^(?:has|is|of)(?=[\s#()\[\]\{\}"'<>?_+\-0-9])/;
this._n3Id = /^id(?=[\s#<])/;
this._shortPredicates = /^a(?=[\s#()\[\]\{\}"'<>])/;
this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)[ \t]*/;
this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)([ \t]*)/;
this._whitespace = /^[ \t]+/;
options = options || {};

Expand Down Expand Up @@ -89,7 +89,7 @@ export default class N3Lexer {
_tokenizeToEnd(callback, inputFinished) {
// Continue parsing as far as possible; the loop will return eventually
let input = this._input;
let currentLineLength = input.length;
let currentLineLength = this._linePosition + input.length;
while (true) {
// Consume one separator line at a time, including its following indentation.
while (true) {
Expand All @@ -103,34 +103,49 @@ export default class N3Lexer {
if (charCode === HASH) {
const comment = this._commentLine.exec(input);
if (comment) {
const commentLength = comment[0].length;
// Keep a trailing CR buffered in case the next chunk starts with LF.
if (!inputFinished && commentLength === input.length &&
input.charCodeAt(commentLength - 1) === CR) {
this._linePosition = currentLineLength - input.length;
return this._input = input;
}
if (this.comments)
emitToken('comment', comment[1], '', this._line, comment[0].length);
input = input.slice(comment[0].length);
currentLineLength = input.length;
emitComment(comment[1], this._line, separatorLength);
input = input.slice(commentLength);
currentLineLength = input.length + comment[2].length;
this._line++;
}
else {
// A comment without a line ending stays buffered until EOF.
input = input.slice(separatorLength);
if (!inputFinished)
if (!inputFinished) {
this._linePosition = currentLineLength - input.length;
return this._input = input;
}
if (this.comments)
emitToken('comment', input.slice(1), '', this._line, input.length);
emitComment(input.slice(1), this._line, 0);
input = '';
break;
}
}
else if (charCode === LF || charCode === CR) {
// A CR at the end of a chunk may still be followed by LF.
if (!inputFinished && charCode === CR && separatorLength + 1 === input.length) {
this._linePosition = currentLineLength - input.length;
return this._input = input;
}
separatorLength += charCode === CR && input.charCodeAt(separatorLength + 1) === LF ? 2 : 1;
// Indentation is part of the same separator match as the newline.
// Indentation is consumed with the newline, but belongs to the next line's columns.
let indentationLength = 0;
const next = input.charCodeAt(separatorLength);
if (next === SPACE || next === TAB) {
const following = input.charCodeAt(separatorLength + 1);
separatorLength += following === SPACE || following === TAB ?
indentationLength = following === SPACE || following === TAB ?
this._whitespace.exec(input.slice(separatorLength))[0].length : 1;
}
input = input.slice(separatorLength);
currentLineLength = input.length;
input = input.slice(separatorLength + indentationLength);
currentLineLength = input.length + indentationLength;
this._line++;
}
else {
Expand All @@ -144,13 +159,15 @@ export default class N3Lexer {
input = null;
emitToken('eof', '', '', this._line, 0);
}
this._linePosition = currentLineLength;
return this._input = input;
}

// Look for specific token types based on the first character
const line = this._line, firstChar = input[0];
let type = '', value = '', prefix = '',
match = null, matchLength = 0, inconclusive = false;
match = null, matchLength = 0, lexicalLength = 0,
finalLineLength = 0, inconclusive = false;
switch (firstChar) {
case '^':
// We need at least 3 tokens lookahead to distinguish ^^<IRI> and ^^pre:fixed
Expand All @@ -177,14 +194,17 @@ export default class N3Lexer {
// Fall through in case the type is an IRI
case '<':
// Try to find a full IRI without escape sequences
if (match = this._unescapedIri.exec(input))
if (match = this._unescapedIri.exec(input)) {
type = 'IRI', value = match[1];
lexicalLength = match[1].length + 2;
}
// Try to find a full IRI with escape sequences
else if (match = this._iri.exec(input)) {
value = this._unescape(match[1], stringEscapeReplacements);
if (value === null || illegalIriChars.test(value))
return reportSyntaxError(this);
type = 'IRI';
lexicalLength = match[1].length + 2;
}
// Try to find a triple term
else if (input.length > 2 && input[1] === '<' && input[2] === '(')
Expand Down Expand Up @@ -214,8 +234,10 @@ export default class N3Lexer {
// we always need a non-dot character before deciding it is a blank node.
// Therefore, try inserting a space if we're at the end of the input.
if ((match = this._blank.exec(input)) ||
inputFinished && (match = this._blank.exec(`${input} `)))
inputFinished && (match = this._blank.exec(`${input} `))) {
type = 'blank', prefix = '_', value = match[1];
lexicalLength = match[1].length + 2;
}
break;

case '"':
Expand All @@ -224,7 +246,7 @@ export default class N3Lexer {
value = match[1];
// Try to find a literal wrapped in three pairs of quotes
else {
({ value, matchLength } = this._parseLiteral(input));
({ value, matchLength, finalLineLength } = this._parseLiteral(input));
if (value === null)
return reportSyntaxError(this);
}
Expand All @@ -241,7 +263,7 @@ export default class N3Lexer {
value = match[1];
// Try to find a literal wrapped in three pairs of quotes
else {
({ value, matchLength } = this._parseLiteral(input));
({ value, matchLength, finalLineLength } = this._parseLiteral(input));
if (value === null)
return reportSyntaxError(this);
}
Expand Down Expand Up @@ -436,8 +458,11 @@ export default class N3Lexer {
// we always need a non-dot character before deciding it is a prefixed name.
// Therefore, try inserting a space if we're at the end of the input.
else if ((match = this._prefixed.exec(input)) ||
inputFinished && (match = this._prefixed.exec(`${input} `)))
type = 'prefixed', prefix = match[1] || '', value = this._unescape(match[2], localNameEscapeReplacements);
inputFinished && (match = this._prefixed.exec(`${input} `))) {
type = 'prefixed', prefix = match[1] || '';
value = this._unescape(match[2], localNameEscapeReplacements);
lexicalLength = prefix.length + match[2].length + 1;
}
}

// A type token is special: it can only be emitted after an IRI or prefixed name is read
Expand All @@ -456,20 +481,44 @@ export default class N3Lexer {
// One exception: error on an unaccounted linebreak (= not inside a triple-quoted literal).
if (inputFinished || (!/^'''|^"""/.test(input) && /\n|\r/.test(input)))
return reportSyntaxError(this);
else
else {
this._linePosition = currentLineLength - input.length;
return this._input = input;
}
}

// Emit the parsed token
// Consumption includes separator whitespace; lexicalLength excludes it
// and any synthetic EOF space. slice below clamps consumption to the input.
const length = matchLength || match[0].length;
const token = emitToken(type, value, prefix, line, length);
let token;
if (finalLineLength) {
token = {
type, value, prefix, line,
start: currentLineLength - input.length,
end: finalLineLength, endLine: this._line,
};
callback(null, token);
}
else
token = emitToken(type, value, prefix, line, lexicalLength || length);
this.previousToken = token;
this._previousMarker = type;

// Advance to next part to tokenize
input = input.slice(length);
if (finalLineLength)
currentLineLength = input.length + finalLineLength;
}

// Emits a comment at its exact position within matched whitespace.
function emitComment(value, line, offset) {
const start = currentLineLength - input.length + offset;
callback(null, {
type: 'comment', value, prefix: '', line,
start, end: start + value.length + 1,
});
}
// Emits the token through the callback
function emitToken(type, value, prefix, line, length) {
const start = input ? currentLineLength - input.length : currentLineLength;
Expand Down Expand Up @@ -568,21 +617,23 @@ export default class N3Lexer {
// means these are actual, non-escaped closing quotes
if (backslashCount % 2 === 0) {
// Extract and unescape the value
const raw = input.substring(openingLength, closingPos);
const lines = raw.split(/\r\n|\r|\n/).length - 1;
const raw = input.substring(openingLength, closingPos),
lines = raw.split(/\r\n|\r|\n/),
lineCount = lines.length - 1;
const matchLength = closingPos + openingLength;
// Only triple-quoted strings can be multi-line
if (openingLength === 1 && lines !== 0 ||
if (openingLength === 1 && lineCount !== 0 ||
openingLength === 3 && this._lineMode)
break;
this._line += lines;
return { value: this._unescape(raw, stringEscapeReplacements), matchLength };
this._line += lineCount;
const finalLineLength = lineCount === 0 ? 0 : lines[lines.length - 1].length + openingLength;
return { value: this._unescape(raw, stringEscapeReplacements), matchLength, finalLineLength };
}
closingPos++;
}
this._literalClosingPos = input.length - openingLength + 1;
}
return { value: '', matchLength: 0 };
return { value: '', matchLength: 0, finalLineLength: 0 };
}

// ### `_syntaxError` creates a syntax error for the given issue
Expand All @@ -599,18 +650,30 @@ export default class N3Lexer {

// ### Strips off any starting UTF BOM mark.
_readStartingBom(input) {
return input.startsWith('\ufeff') ? input.slice(1) : input;
if (input.startsWith('\ufeff')) {
this._linePosition = 1;
return input.slice(1);
}
return input;
}

// ## Public methods

// ### `tokenize` starts the transformation of an N3 document into an array of tokens.
// The input can be a string or a stream.
// Token ranges use one-based lines and zero-based, end-exclusive UTF-16 columns.
// Separator whitespace counts towards the next token's start, outside either range.
// Multiline tokens also have endLine; their end column is relative to that line.
tokenize(input, callback) {
// Deferred tokenization and stream events can outlive their invocation.
// Ignore them once a later call takes ownership of the lexer state.
const tokenization = this._tokenization = {};
this._line = 1;
this._linePosition = 0;
this._previousMarker = undefined;
this.previousToken = undefined;
this._literalClosingPos = 0;
this._input = undefined;

// If the input is a string, continuously emit tokens through the callback until the end
if (typeof input === 'string') {
Expand Down
Loading