Skip to content

perf(provenance): prototype optimized side table #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jeswr wants to merge 11 commits into from
Closed

perf(provenance): prototype optimized side table #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f628426
Add opt-in term source spans and a provenance-tracking parser wrapper
Aug 3, 2026
38d57a9
scripts: derive the provenance instrumentation mechanically
Aug 3, 2026
e920d8e
Oxford comma in comment
ericprud Aug 9, 2026
a197fa8
Merge remote-tracking branch 'upstream/main' into term-provenance
Aug 9, 2026
ef33071
fix: span predicate literals introduced by upstream #613
Aug 9, 2026
64da0b1
fix: span pre-datatyped literals (numbers and booleans)
Aug 9, 2026
6ae0e5e
chore: narrow the _readTripleTerm anchor to the calls it rewrites
Aug 9, 2026
113dd60
Merge remote-tracking branch 'origin/main' into codex/provenance-pr-base
jeswr Sep 1, 2026
742fb64
Prototype provenance with an optimized parser side table
jeswr Sep 1, 2026
8e61263
Remove stale provenance patch generator
jeswr Sep 1, 2026
d67b7cc
Complete side-table provenance coverage
jeswr Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/N3Lexer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@
this.comments = !!options.comments;
// Cache the last tested closing position of long literals
this._literalClosingPos = 0;
// Absolute character offset of the head of `_input` within the document
this._offset = 0;
// Only annotate tokens with absolute offsets when asked (keeps the
// default token shape unchanged)
this._trackOffsets = !!options.trackOffsets;
}

// ## Private methods
Expand All @@ -94,6 +99,7 @@
_tokenizeToEnd(callback, inputFinished) {
// Continue parsing as far as possible; the loop will return eventually
let input = this._input;
const self = this;
let currentLineLength = input.length;
while (true) {
// Count and skip whitespace lines
Expand All @@ -103,12 +109,14 @@
if (this.comments && (comment = this._comment.exec(whiteSpaceMatch[0])))
emitToken('comment', comment[1], '', this._line, whiteSpaceMatch[0].length);
// Advance the input
this._offset += whiteSpaceMatch[0].length;
input = input.slice(whiteSpaceMatch[0].length);
currentLineLength = input.length;
this._line++;
}
// Skip whitespace on current line
if (!whiteSpaceMatch && (whiteSpaceMatch = this._whitespace.exec(input)))
this._offset += whiteSpaceMatch[0].length,
input = input.slice(whiteSpaceMatch[0].length);

// Stop for now if we're at the end
Expand Down Expand Up @@ -137,7 +145,7 @@
else if (input[1] === '^') {
this._previousMarker = '^^';
// Move to type IRI or prefixed name
input = input.slice(2);
this._offset += 2, input = input.slice(2);
if (input[0] !== '<') {
inconclusive = true;
break;
Expand Down Expand Up @@ -283,8 +291,8 @@
// Try to find a number. Since it can contain (but not end with) a dot,
// we always need a non-dot character before deciding it is a number.
// Therefore, try inserting a space if we're at the end of the input.
if (match = this._number.exec(input) ||

Check failure

Code scanning / CodeQL

Polynomial regular expression used on uncontrolled data High

This
regular expression
Loading
that depends on
library input
Loading
may run slow on strings with many repetitions of '9'.
inputFinished && (match = this._number.exec(`${input} `))) {

Check failure

Code scanning / CodeQL

Polynomial regular expression used on uncontrolled data High

This
regular expression
Loading
that depends on
library input
Loading
may run slow on strings with many repetitions of '9'.
type = 'literal', value = match[0];
prefix = (typeof match[1] === 'string' ? xsd.double :
(typeof match[2] === 'string' ? xsd.decimal : xsd.integer));
Expand Down Expand Up @@ -440,14 +448,19 @@
this._previousMarker = type;

// Advance to next part to tokenize
input = input.slice(length);
this._offset += length, input = input.slice(length);
}

// Emits the token through the callback
function emitToken(type, value, prefix, line, length) {
const start = input ? currentLineLength - input.length : currentLineLength;
const end = start + length;
const token = { type, value, prefix, line, start, end };
// absolute document offsets (the token begins at the unconsumed head)
if (self._trackOffsets) {
token.offsetStart = self._offset;
token.offsetEnd = self._offset + length;
}
callback(null, token);
return token;
}
Expand Down
91 changes: 59 additions & 32 deletions src/N3Parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@ export default class N3Parser {
options = options || {};
this._setBase(options.baseIRI);
options.factory && initDataFactory(this, options.factory);
// Opt-in source-span tracking: after each emitted quad, invoke
// onQuadSpans(quad, {subject, predicate, object, graph}) with the
// lexer's {line, start, end} for each position's source token (null for
// synthetic terms without one). Zero cost when the option is absent.
this._onQuadSpans = options.onQuadSpans || null;
this._termSpans = this._onQuadSpans ? new Map() : null;

// Set supported features depending on the format
const format = (typeof options.format === 'string') ?
Expand All @@ -39,7 +45,7 @@ export default class N3Parser {
this._resolveRelativeIRI = iri => { return null; };
this._blankNodePrefix = typeof options.blankNodePrefix !== 'string' ? '' :
options.blankNodePrefix.replace(/^(?!_:)/, '_:');
this._lexer = options.lexer || new N3Lexer({ lineMode: isLineMode, n3: isN3, isImpliedBy: this._isImpliedBy });
this._lexer = options.lexer || new N3Lexer({ lineMode: isLineMode, n3: isN3, isImpliedBy: this._isImpliedBy, trackOffsets: !!options.onQuadSpans });
// Disable explicit quantifiers by default
this._explicitQuantifiers = !!options.explicitQuantifiers;
// Disable parsing of unsupported versions by default
Expand Down Expand Up @@ -249,7 +255,7 @@ export default class N3Parser {
// In N3 mode, replace the entity if it is quantified
if (!quantifier && this._n3Mode && (value.id in this._quantified))
value = this._quantified[value.id];
return value;
return this._noteSpan(value, token);
}

// ### `_readList` starts reading a list in the subject, predicate, or object position
Expand All @@ -274,7 +280,7 @@ export default class N3Parser {
case '[':
// Start a new quad with a new blank node as subject
this._saveContext('blank', this._graph,
this._subject = this._factory.blankNode(), null, null);
this._subject = this._noteSpan(this._factory.blankNode(), token), null, null);
return this._readBlankNodeHead;
case '(':
return this._readList(token, this.RDF_NIL, null, null);
Expand All @@ -283,7 +289,7 @@ export default class N3Parser {
if (!this._n3Mode)
return this._error('Unexpected graph', token);
this._saveContext('formula', this._graph,
this._graph = this._factory.blankNode(), null, null);
this._graph = this._noteSpan(this._factory.blankNode(), token), null, null);
return this._readInFormulaContext;
case '}':
// No subject; the graph in which we are reading is closed instead
Expand All @@ -308,10 +314,11 @@ export default class N3Parser {

if (token.prefix.length === 0) {
this._literalValue = token.value;
if (this._termSpans !== null) this._literalSpan = token;
return this._completeSubjectLiteral;
}
else {
this._subject = this._factory.literal(token.value, this._factory.namedNode(token.prefix));
this._subject = this._noteSpan(this._factory.literal(token.value, this._factory.namedNode(token.prefix)), token);
// This branch is N3-only, so the literal subject might start a path
return this._getPathReader(this._readPredicateOrNamedGraph);
}
Expand Down Expand Up @@ -378,10 +385,11 @@ export default class N3Parser {

if (token.prefix.length === 0) {
this._literalValue = token.value;
if (this._termSpans !== null) this._literalSpan = token;
return this._completePredicateLiteral;
}
else
this._predicate = this._factory.literal(token.value, this._factory.namedNode(token.prefix));
this._predicate = this._noteSpan(this._factory.literal(token.value, this._factory.namedNode(token.prefix)), token);

pathable = true;
break;
Expand All @@ -394,7 +402,7 @@ export default class N3Parser {
if (this._n3Mode) {
// Start a new quad with a new blank node as subject
this._saveContext('blank', this._graph, this._subject,
this._subject = this._factory.blankNode(), null);
this._subject = this._noteSpan(this._factory.blankNode(), token), null);
return this._readBlankNodeHead;
}
return this._error('Disallowed blank node as predicate', token);
Expand Down Expand Up @@ -440,11 +448,12 @@ export default class N3Parser {
// Regular literal, can still get a datatype or language
if (token.prefix.length === 0) {
this._literalValue = token.value;
if (this._termSpans !== null) this._literalSpan = token;
return this._readDataTypeOrLang;
}
// Pre-datatyped string literal (prefix stores the datatype)
else {
this._object = this._factory.literal(token.value, this._factory.namedNode(token.prefix));
this._object = this._noteSpan(this._factory.literal(token.value, this._factory.namedNode(token.prefix)), token);
// In N3 mode, the literal object might start a path
if (this._n3Mode)
return this._getPathReader(this._getContextEndReader());
Expand All @@ -453,7 +462,7 @@ export default class N3Parser {
case '[':
// Start a new quad with a new blank node as subject
this._saveContext('blank', this._graph, this._subject, this._predicate,
this._subject = this._factory.blankNode());
this._subject = this._noteSpan(this._factory.blankNode(), token));
return this._readBlankNodeHead;
case '(':
return this._readList(token, this._subject, this._predicate, this.RDF_NIL);
Expand All @@ -462,7 +471,7 @@ export default class N3Parser {
if (!this._n3Mode)
return this._error('Unexpected graph', token);
this._saveContext('formula', this._graph, this._subject, this._predicate,
this._graph = this._factory.blankNode());
this._graph = this._noteSpan(this._factory.blankNode(), token));
return this._readInFormulaContext;
case '<<(':
this._saveContext('<<(', this._graph, this._subject, this._predicate, null);
Expand Down Expand Up @@ -593,14 +602,14 @@ export default class N3Parser {
case '[':
// Stack the current list quad and start a new quad with a blank node as subject
this._saveContext('blank', this._graph,
list = this._factory.blankNode(), this.RDF_FIRST,
this._subject = item = this._factory.blankNode());
list = this._noteSpan(this._factory.blankNode(), token), this.RDF_FIRST,
this._subject = item = this._noteSpan(this._factory.blankNode(), token));
next = this._readBlankNodeHead;
break;
case '(':
// Stack the current list quad and start a new list
this._saveContext('list', this._graph,
list = this._factory.blankNode(), this.RDF_FIRST, this.RDF_NIL);
list = this._noteSpan(this._factory.blankNode(), token), this.RDF_FIRST, this.RDF_NIL);
this._subject = null;
break;
case ')':
Expand Down Expand Up @@ -655,11 +664,12 @@ export default class N3Parser {
// Regular literal, can still get a datatype or language
if (token.prefix.length === 0) {
this._literalValue = token.value;
if (this._termSpans !== null) this._literalSpan = token;
next = this._readListItemDataTypeOrLang;
}
// Pre-datatyped string literal (prefix stores the datatype)
else {
item = this._factory.literal(token.value, this._factory.namedNode(token.prefix));
item = this._noteSpan(this._factory.literal(token.value, this._factory.namedNode(token.prefix)), token);
next = this._getContextEndReader();
}
break;
Expand All @@ -669,8 +679,8 @@ export default class N3Parser {
return this._error('Unexpected graph', token);
// The formula is an item of the list,
// so it must be linked in the list's graph before the graph changes
list = this._factory.blankNode();
item = this._factory.blankNode();
list = this._noteSpan(this._factory.blankNode(), token);
item = this._noteSpan(this._factory.blankNode(), token);
// Is this the first element of the list?
if (previousList === null) {
// This list is either the subject or the object of its parent
Expand Down Expand Up @@ -707,7 +717,7 @@ export default class N3Parser {

// Create a new blank node if no item head was assigned yet
if (list === null)
this._subject = list = this._factory.blankNode();
this._subject = list = this._noteSpan(this._factory.blankNode(), token);

// When reading a reified triple or triple term, store the list as subject in the stack, as this will be overridden when reading the triple.
if (token.type === '<<' || token.type === '<<(')
Expand Down Expand Up @@ -772,14 +782,14 @@ export default class N3Parser {
if (datatype.value === namespaces.rdf.langString || datatype.value === namespaces.rdf.dirLangString) {
return this._error('Detected illegal (directional) languaged-tagged string with explicit datatype', token);
}
literal = this._factory.literal(this._literalValue, datatype);
literal = this._noteLiteralSpan(this._factory.literal(this._literalValue, datatype));
token = null;
break;
// Create a language-tagged string
case 'langcode':
if (token.value.split('-').some(t => t.length > 8))
return this._error('Detected language tag with subtag longer than 8 characters', token);
literal = this._factory.literal(this._literalValue, token.value);
literal = this._noteLiteralSpan(this._factory.literal(this._literalValue, token.value));
this._literalLanguage = token.value;
token = null;
// Save state for a possible direction tag
Expand All @@ -788,7 +798,7 @@ export default class N3Parser {
break;
// Create a simple string literal by default
default:
literal = this._factory.literal(this._literalValue);
literal = this._noteLiteralSpan(this._factory.literal(this._literalValue));
}

return { token, literal, readCb };
Expand All @@ -799,7 +809,7 @@ export default class N3Parser {
const component = this._literalComponent, listItem = this._literalListItem;
// Attempt to read a dircode
if (token.type === 'dircode') {
const term = this._factory.literal(this._literalValue, { language: this._literalLanguage, direction: token.value });
const term = this._noteLiteralSpan(this._factory.literal(this._literalValue, { language: this._literalLanguage, direction: token.value }));
if (component === 'subject')
this._subject = term;
else if (component === 'predicate')
Expand Down Expand Up @@ -1117,7 +1127,7 @@ export default class N3Parser {
_readNamedGraphBlankLabel(token) {
if (token.type !== ']')
return this._error('Invalid graph label', token);
this._subject = this._factory.blankNode();
this._subject = this._noteSpan(this._factory.blankNode(), token);
return this._readGraph;
}

Expand Down Expand Up @@ -1147,17 +1157,17 @@ export default class N3Parser {
}
// Without explicit quantifiers, map entities to a quantified entity
if (!this._explicitQuantifiers)
this._quantified[entity.id] = this._factory[this._quantifier](this._factory.blankNode().value);
this._quantified[entity.id] = this._factory[this._quantifier](this._noteSpan(this._factory.blankNode(), token).value);
// With explicit quantifiers, output the reified quantifier
else {
// If this is the first item, start a new quantifier list
if (this._subject === null)
this._emit(this._graph || this.DEFAULTGRAPH, this._predicate,
this._subject = this._factory.blankNode(), this.QUANTIFIERS_GRAPH);
this._subject = this._noteSpan(this._factory.blankNode(), token), this.QUANTIFIERS_GRAPH);
// Otherwise, continue the previous list
else
this._emit(this._subject, this.RDF_REST,
this._subject = this._factory.blankNode(), this.QUANTIFIERS_GRAPH);
this._subject = this._noteSpan(this._factory.blankNode(), token), this.QUANTIFIERS_GRAPH);
// Output the list item
this._emit(this._subject, this.RDF_FIRST, entity, this.QUANTIFIERS_GRAPH);
}
Expand Down Expand Up @@ -1229,7 +1239,7 @@ export default class N3Parser {
// ### `_readForwardPath` reads a '!' path
_readForwardPath(token) {
let subject, predicate;
const object = this._factory.blankNode();
const object = this._noteSpan(this._factory.blankNode(), token);
// The next token is the predicate
if ((predicate = this._readEntity(token)) === undefined)
return;
Expand All @@ -1247,7 +1257,7 @@ export default class N3Parser {

// ### `_readBackwardPath` reads a '^' path
_readBackwardPath(token) {
const subject = this._factory.blankNode();
const subject = this._noteSpan(this._factory.blankNode(), token);
let predicate, object;
// The next token is the predicate
if ((predicate = this._readEntity(token)) === undefined)
Expand All @@ -1269,8 +1279,8 @@ export default class N3Parser {
if (token.type !== ')>>')
return this._error(`Expected )>> but got ${token.type}`, token);
// Read the quad and restore the previous context
const quad = this._factory.quad(this._subject, this._predicate, this._object,
this._graph || this.DEFAULTGRAPH);
const quad = this._noteSpan(this._factory.quad(this._subject, this._predicate, this._object,
this._graph || this.DEFAULTGRAPH), token);
this._restoreContext('<<(', token);

// If we're in a list, continue processing that list
Expand Down Expand Up @@ -1386,9 +1396,9 @@ export default class N3Parser {
_readTripleTerm() {
const stack = this._contextStack, parent = stack.length && stack[stack.length - 1];
const parentGraph = parent ? parent.graph : undefined;
const reifier = this._reifier || this._factory.blankNode();
const reifier = this._reifier || this._noteSpan(this._factory.blankNode(), null);
this._reifier = null;
this._tripleTerm = this._tripleTerm || this._factory.quad(this._subject, this._predicate, this._object);
this._tripleTerm = this._tripleTerm || this._noteSpan(this._factory.quad(this._subject, this._predicate, this._object), null);
this._emit(reifier, this.RDF_REIFIES, this._tripleTerm, parentGraph || this._graph || this.DEFAULTGRAPH);
return reifier;
}
Expand All @@ -1415,7 +1425,24 @@ export default class N3Parser {

// ### `_emit` sends a quad through the callback
_emit(subject, predicate, object, graph) {
this._callback(null, this._factory.quad(subject, predicate, object, graph || this.DEFAULTGRAPH));
const quad = this._factory.quad(subject, predicate, object, graph || this.DEFAULTGRAPH);
if (this._onQuadSpans)
this._onQuadSpans(quad);
this._callback(null, quad);
}

// ### `_noteLiteralSpan` records the pending string token's span
_noteLiteralSpan(literal) {
if (this._termSpans !== null && this._literalSpan)
this._termSpans.set(literal, this._literalSpan);
return literal;
}

// ### `_noteSpan` records a term's source token span when tracking is on
_noteSpan(term, token) {
if (this._termSpans !== null && term && token && typeof term === 'object')
this._termSpans.set(term, token);
return term;
}

// ### `_error` emits an error message through the callback
Expand Down
Loading