From 9e38b2b757c5f831a3eb343e78fb6b6ae0f086d4 Mon Sep 17 00:00:00 2001 From: Matthew Date: Sat, 8 Aug 2026 23:29:20 +0100 Subject: [PATCH 01/11] chore: Setup SonarQube for IDE --- .gitignore | 6 ++++++ .sonarlint/RedSeaModernLanguage.json | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index be19bc4..86fe890 100644 --- a/.gitignore +++ b/.gitignore @@ -399,3 +399,9 @@ ADMONITION.md docs/_site/ docs/__site/ docs/api/ + +# SonarQube has config for SonarScanner and stuff - dont track it +.sonarqube + +# SonarLint is team configuration for SonarQube for IDE - track it +!.sonarlint diff --git a/.sonarlint/RedSeaModernLanguage.json b/.sonarlint/RedSeaModernLanguage.json index 520cef0..f854d17 100644 --- a/.sonarlint/RedSeaModernLanguage.json +++ b/.sonarlint/RedSeaModernLanguage.json @@ -1,5 +1,5 @@ { - "sonarCloudOrganization": "oas", - "region": "EU", - "projectKey": "oas_RedSeaModernLanguage" + "sonarCloudOrganization": "oas", + "projectKey": "oas_RedSeaModernLanguage", + "region": "EU" } \ No newline at end of file From bb0dee1c933ceeb926bb94d9f57baa9ddd44ff4a Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 9 Aug 2026 00:51:56 +0100 Subject: [PATCH 02/11] chore: Untrack VSCode tasks --- .gitignore | 8 +++--- .vscode/tasks.json | 61 ---------------------------------------------- 2 files changed, 3 insertions(+), 66 deletions(-) delete mode 100644 .vscode/tasks.json diff --git a/.gitignore b/.gitignore index 86fe890..88a9033 100644 --- a/.gitignore +++ b/.gitignore @@ -378,9 +378,7 @@ _ReSharper*/ *__NOTYET # Configs -.vscode/* -!.vscode/tasks.json -!.vscode/enable.settings.jsonc +.vscode/ # Site (on main) site/ @@ -401,7 +399,7 @@ docs/__site/ docs/api/ # SonarQube has config for SonarScanner and stuff - dont track it -.sonarqube +.sonarqube/ # SonarLint is team configuration for SonarQube for IDE - track it -!.sonarlint +!.sonarlint/ diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index 3c3b0d7..0000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "label": "Build Docs", - "type": "shell", - "command": "mkdocs build --strict" - }, - { - "label": "Clean Docs", - "type": "shell", - "command": "mkdocs build --strict --clean" - }, - { - "label": "Serve Docs (Clean)", - "type": "shell", - "command": "mkdocs serve -s -c -o" - }, - { - "label": "Serve Docs (Dirty)", - "type": "shell", - "command": "mkdocs serve -s --dirty -o" - }, - { - "label": "build", - "command": "dotnet", - "type": "process", - "args": [ - "build", - "${workspaceFolder}/src/RSML.CLI/RSML.CLI.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary;ForceNoAlign" - ], - "problemMatcher": "$msCompile" - }, - { - "label": "publish", - "command": "dotnet", - "type": "process", - "args": [ - "publish", - "${workspaceFolder}/src/RSML.CLI/RSML.CLI.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary;ForceNoAlign" - ], - "problemMatcher": "$msCompile" - }, - { - "label": "watch", - "command": "dotnet", - "type": "process", - "args": [ - "watch", - "run", - "--project", - "${workspaceFolder}/src/RSML.CLI/RSML.CLI.csproj" - ], - "problemMatcher": "$msCompile" - } - ] -} From cc18daf1efbf0c3be3d86edd528031138f026d5b Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 9 Aug 2026 00:53:46 +0100 Subject: [PATCH 03/11] refactor(buffer): Replace locations and source spans with built-in Index and Range types --- src/RSML.Language.Lexing/BufferLexer.cs | 33 +-- src/RSML.Language.Lexing/Tokens/Token.cs | 8 +- .../Diagnostics/Diagnostic.cs | 67 +++--- .../{CharacterExtensions.cs => Extensions.cs} | 2 +- .../GlobalSuppressions.cs | 1 - .../Sources/IBuffer.cs | 74 +++---- .../Sources/SourceLocation.cs | 125 ------------ .../Sources/SourceSpan.cs | 100 --------- .../GlobalSuppressions.cs | 1 - .../ReadOnlyStringBuffer.cs | 159 +++++++-------- .../Sources/ReadOnlyStringBufferTests.cs | 190 +++++++----------- 11 files changed, 242 insertions(+), 518 deletions(-) rename src/RSML.Toolchain.Abstractions/{CharacterExtensions.cs => Extensions.cs} (97%) delete mode 100644 src/RSML.Toolchain.Abstractions/Sources/SourceLocation.cs delete mode 100644 src/RSML.Toolchain.Abstractions/Sources/SourceSpan.cs diff --git a/src/RSML.Language.Lexing/BufferLexer.cs b/src/RSML.Language.Lexing/BufferLexer.cs index 508eb11..c92eaa6 100644 --- a/src/RSML.Language.Lexing/BufferLexer.cs +++ b/src/RSML.Language.Lexing/BufferLexer.cs @@ -32,10 +32,9 @@ public override Result GetNextToken() SkipWhitespaceAndComments(); if (cursor >= buffer.Length) - return Result.Success(new Token(TokenKind.Eof, null, SourceSpan.Empty)); - - var startLoc = buffer.GetSourceLocation(cursor); + return Result.Success(new Token(TokenKind.Eof, null, new())); + int startLoc = cursor; char c = buffer[cursor]; // strings @@ -51,7 +50,7 @@ public override Result GetNextToken() return ScanIdentifierOrKeyword(startLoc); if (c == '.') - return Result.Success(new Token(TokenKind.MemberAccess, null, new(startLoc, buffer.GetSourceLocation(++cursor)))); + return Result.Success(new Token(TokenKind.MemberAccess, null, new(startLoc, ++cursor))); // todo: add the remaining possible paths return Result.Failure(new(LexerErrorCodes.FailedToLexToken, "Tried all possible token logic paths, but none was true.", Severity.Error)); @@ -82,7 +81,7 @@ public override IEnumerable Lex() } } - private Result ScanNumber(SourceLocation startLoc) + private Result ScanNumber(int startLoc) { bool dot = false; @@ -91,7 +90,7 @@ private Result ScanNumber(SourceLocation startLoc) if (buffer[cursor] == '.') { if (dot) - return Result.Success(new Token(TokenKind.Number, null, new(startLoc, buffer.GetSourceLocation(cursor - 1)))); + return Result.Success(new Token(TokenKind.Number, null, new(startLoc, cursor))); else dot = true; @@ -100,10 +99,10 @@ private Result ScanNumber(SourceLocation startLoc) cursor++; } - return Result.Success(new Token(TokenKind.Number, null, new(startLoc, buffer.GetSourceLocation(cursor)))); + return Result.Success(new Token(TokenKind.Number, null, new(startLoc, cursor))); } - private Result ScanStringLiteral(SourceLocation startLoc) + private Result ScanStringLiteral(int startLoc) { cursor++; bool escaping = false; @@ -114,7 +113,8 @@ private Result ScanStringLiteral(SourceLocation startLoc) { return Result.Failure(new( LexerErrorCodes.UnterminatedStringLiteral, - new SourceSpan(startLoc, new(cursor, startLoc.Line, cursor - startLoc.Index + startLoc.Column)), + buffer.GetLocationDetails((Index)startLoc), + buffer.GetLocationDetails((Index)cursor), "A string literal must begin and end in the same line.", Severity.Error )); @@ -132,24 +132,25 @@ private Result ScanStringLiteral(SourceLocation startLoc) if (cursor < buffer.Length) cursor++; // skip end quote if anything beyond it - return Result.Success(new Token(TokenKind.StringLiteral, null, new(startLoc, buffer.GetSourceLocation(cursor)))); + return Result.Success(new Token(TokenKind.StringLiteral, null, startLoc..cursor)); } - private Result ScanIdentifierOrKeyword(SourceLocation startLoc) + private Result ScanIdentifierOrKeyword(int startLoc) { while (cursor < buffer.Length && (Char.IsAsciiLetterOrDigit(buffer[cursor]) || buffer[cursor] == '_')) cursor++; - SourceSpan span = new(startLoc, buffer.GetSourceLocation(cursor)); + Range range = startLoc..cursor; - if (Keywords.Contains(buffer[span])) + if (Keywords.Contains(buffer[range])) { - var token = new Token(GetKeywordTokenKind(buffer[span]), null, span); // is keyword + var token = new Token(GetKeywordTokenKind(buffer[range]), null, range); // is keyword return token.Kind == TokenKind.Unknown ? Result.Failure(new( LexerErrorCodes.FailedToIdentifyKeyword, - new SourceSpan(startLoc, new(cursor, startLoc.Line, cursor - startLoc.Index + startLoc.Column)), + buffer.GetLocationDetails(startLoc), + buffer.GetLocationDetails(cursor), "Despite identifying the object in question as a keyword, the lexer failed to resolve exactly which keyword it was." + "This likely means the keyword in question is reserved for future use, but isn't implemented yet.", Severity.Error @@ -158,7 +159,7 @@ private Result ScanIdentifierOrKeyword(SourceLocation startLoc) } else { - return Result.Success(new Token(TokenKind.Identifier, null, span)); // is identifier + return Result.Success(new Token(TokenKind.Identifier, null, range)); // is identifier } } diff --git a/src/RSML.Language.Lexing/Tokens/Token.cs b/src/RSML.Language.Lexing/Tokens/Token.cs index e06cb65..ec16a92 100644 --- a/src/RSML.Language.Lexing/Tokens/Token.cs +++ b/src/RSML.Language.Lexing/Tokens/Token.cs @@ -1,4 +1,4 @@ -using OceanApocalypse.RSML.Toolchain.Abstractions.Sources; +using System; namespace OceanApocalypse.RSML.Language.Lexing.Tokens; @@ -8,11 +8,11 @@ namespace OceanApocalypse.RSML.Language.Lexing.Tokens; /// /// An integer that identifies the type of token. /// The token's value. -/// The span where the token occurs. -public record struct Token(TokenKind Kind, object? Value, SourceSpan Span) +/// The range where the token occurs. +public record struct Token(TokenKind Kind, object? Value, Range Range) { /// /// Empty token. Used when something goes wrong. /// - public readonly static Token Empty = new(TokenKind.Unknown, null, SourceSpan.Empty); + public readonly static Token Empty = new(TokenKind.Unknown, null, new()); } diff --git a/src/RSML.Toolchain.Abstractions/Diagnostics/Diagnostic.cs b/src/RSML.Toolchain.Abstractions/Diagnostics/Diagnostic.cs index a2274ab..a13c4ed 100644 --- a/src/RSML.Toolchain.Abstractions/Diagnostics/Diagnostic.cs +++ b/src/RSML.Toolchain.Abstractions/Diagnostics/Diagnostic.cs @@ -12,9 +12,14 @@ namespace OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; public readonly struct Diagnostic : IFormattable, IEquatable { /// - /// The span the error relates to. + /// The start index the error relates to (inclusive). /// - public SourceSpan Span { get; } + public (Index Index, int Line, int Column) Start { get; } = (0, 0, 0); + + /// + /// The end index the error relates to (exclusive). + /// + public (Index Index, int Line, int Column) End { get; } = (0, 0, 0); /// /// The error's code. Contains information about the category of the error. @@ -44,7 +49,6 @@ public Diagnostic(string code) ThrowIfInvalidErrorCode(code); Code = code; - Span = SourceSpan.Empty; Message = ""; Severity = Severity.None; } @@ -58,7 +62,6 @@ public Diagnostic(string code, string message) ThrowIfInvalidErrorCode(code); Code = code; - Span = SourceSpan.Empty; Message = message; Severity = Severity.None; } @@ -72,7 +75,6 @@ public Diagnostic(string code, Severity severity) ThrowIfInvalidErrorCode(code); Code = code; - Span = SourceSpan.Empty; Message = ""; Severity = severity; } @@ -87,23 +89,24 @@ public Diagnostic(string code, string message, Severity severity) ThrowIfInvalidErrorCode(code); Code = code; - Span = SourceSpan.Empty; Message = message; Severity = severity; } /// Creates a new diagnostic. /// The error code. - /// The span the error relates to. + /// The inclusive start of the range. + /// The exclusive end of the range. /// A brief error message detailing why it has happened. /// The error's severity. - public Diagnostic(string code, SourceSpan span, string message, Severity severity) + public Diagnostic(string code, (Index idx, int line, int col) spanStart, (Index idx, int line, int col) spanEnd, string message, Severity severity) { ArgumentNullException.ThrowIfNullOrWhiteSpace(code); ThrowIfInvalidErrorCode(code); Code = code; - Span = span; + Start = spanStart; + End = spanEnd; Message = message; Severity = severity; } @@ -115,7 +118,7 @@ public override bool Equals( ) => obj is Diagnostic error && Equals(error); /// - public bool Equals(Diagnostic other) => Message == other.Message && Code == other.Code && Severity == other.Severity && Span.Equals(other.Span); + public bool Equals(Diagnostic other) => Message == other.Message && Code == other.Code && Severity == other.Severity && Start.Equals(other.Start) && End.Equals(other.End); /// /// Checks if two s are equal to each other. @@ -130,13 +133,13 @@ public override bool Equals( public static bool operator !=(Diagnostic left, Diagnostic right) => !left.Equals(right); /// - public override int GetHashCode() => unchecked(HashCode.Combine(Span, Code, Message, Severity)); + public override int GetHashCode() => unchecked(HashCode.Combine(Start, End, Code, Message, Severity)); /// /// Returns a generic string representation of the current instance. /// /// The string representation. - public override string ToString() => $"Diagnostic(Code={Code}, Span={Span}, Message={Message}, Severity={Severity})"; + public override string ToString() => $"Diagnostic(Code={Code}, Start={Start}, End={End}, Message={Message}, Severity={Severity})"; /// /// Given a format, tries to return a string that uses said format as a basis for the representation. @@ -153,7 +156,7 @@ public string ToString(string? format, IFormatProvider? formatProvider) case "I": case "INIT": case "NET": - return $"new Diagnostic(\"{Code}\", {Span.ToString("ctor", null)}, \"{Message}\", {Severity})"; + return $"new Diagnostic(\"{Code}\", \"{Start}\", \"{End}\", \"{Message}\", {Severity})"; case "LOG": string prefix = Severity switch @@ -165,21 +168,35 @@ public string ToString(string? format, IFormatProvider? formatProvider) _ => "" }; - if (Span.IsSingleLine) - return $"[{prefix}{Code}] @ L{Span.Start.Line + 1},C({Span.Start.Column + 1}..{Span.End.Column + 1}) : {Message}"; + if (Start.Line == End.Line) + return $"[{prefix}{Code}] @ L{Start.Line + 1},C({Start.Column + 1}..{End.Column + 1}) : {Message}"; - return $"[{prefix}{Code}] @ L({Span.Start.Line + 1}..{Span.End.Line + 1}),C({Span.Start.Column + 1}..{Span.End.Column + 1}) : {Message}"; + return $"[{prefix}{Code}] @ L({Start.Line + 1}..{End.Line + 1}),C({Start.Column + 1}..{End.Column + 1}) : {Message}"; case "JSON": - return - $$""" - { - "errorCode": "{{Code}}", - "span": {{Span.ToString("JSON", null)}}, - "message": "{{Message}}", - "severity": "{{Severity}}" - } - """; + return $$""" + { + "errorCode": "{{Code}}", + "range": [ + { + "index": { + "value": {{Start.Index.Value}}, + "isFromEnd": {{Start.Index.IsFromEnd}} + }, + "line": {{Start.Line}}, + "column": {{Start.Column}} + }, + { + "index": { + "value": {{End.Index.Value}}, + "isFromEnd": {{End.Index.IsFromEnd}} + }, + "line": {{End.Line}}, + "column": {{End.Column}} + } + ] + } + """; default: return ToString(); diff --git a/src/RSML.Toolchain.Abstractions/CharacterExtensions.cs b/src/RSML.Toolchain.Abstractions/Extensions.cs similarity index 97% rename from src/RSML.Toolchain.Abstractions/CharacterExtensions.cs rename to src/RSML.Toolchain.Abstractions/Extensions.cs index 0af8522..9ce4725 100644 --- a/src/RSML.Toolchain.Abstractions/CharacterExtensions.cs +++ b/src/RSML.Toolchain.Abstractions/Extensions.cs @@ -7,7 +7,7 @@ namespace OceanApocalypse.RSML.Toolchain.Abstractions; /// /// Extension members for characters. /// -public static class CharacterExtensions +public static class Extensions { extension(char character) { diff --git a/src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs b/src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs index f5b887b..82aacc3 100644 --- a/src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs +++ b/src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs @@ -5,5 +5,4 @@ using System.Diagnostics.CodeAnalysis; -[assembly: SuppressMessage("Design", "CA1043:Use Integral Or String Argument For Indexers", Justification = "Refers to a single location.", Scope = "member", Target = "~P:OceanApocalypse.RSML.Toolchain.Abstractions.Sources.IBuffer.Item(OceanApocalypse.RSML.Toolchain.Abstractions.Sources.SourceLocation)")] [assembly: SuppressMessage("Style", "IDE0305:Simplify collection initialization", Justification = "Hides an allocation.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics.DiagnosticCollector.GetAll~System.Collections.Immutable.ImmutableArray{OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics.Diagnostic}")] diff --git a/src/RSML.Toolchain.Abstractions/Sources/IBuffer.cs b/src/RSML.Toolchain.Abstractions/Sources/IBuffer.cs index b467bca..d80d1f7 100644 --- a/src/RSML.Toolchain.Abstractions/Sources/IBuffer.cs +++ b/src/RSML.Toolchain.Abstractions/Sources/IBuffer.cs @@ -40,18 +40,18 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable char this[int index] { get; } /// - /// Gets a span of items out of the buffer. + /// Gets a single item out of the buffer. /// - /// The start and end locations of the span to retrieve. - /// The items. - ReadOnlySpan this[SourceSpan span] { get; } + /// The index of the item to retrieve. + /// The item. + char this[Index index] { get; } /// - /// Gets a single item out of the buffer. + /// Gets a span of items out of the buffer. /// - /// The location of the item to retrieve. - /// The item. - char this[SourceLocation location] { get; } + /// The range to retrieve. + /// The items. + ReadOnlySpan this[Range range] { get; } /// /// Counts the amount of items until the next line separator in the buffer, relative to a given . @@ -62,7 +62,7 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable /// Whether the line separator at which the method stopped is the CR in a CRLF sequence. If true, the next item in the buffer is LF. /// /// The index of the next line separator, relative to an . - int CountUntilEndOfLine(int index, out bool isCrLf); + int CountUntilEndOfLine(Index index, out bool isCrLf); /// /// Counts the amount of items until the next non-whitespace item in the buffer, relative to a given . @@ -70,7 +70,7 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable /// /// The index at which to start counting. /// The index of the next non-whitespace item, relative to a . - int CountUntilNotWhitespace(int index); + int CountUntilNotWhitespace(Index index); /// /// Counts the amount of items until the next whitespace item in the buffer, relative to a given . @@ -78,24 +78,24 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable /// /// The index at which to start counting. /// The index of the next whitespace item, relative to a . - int CountUntilWhitespace(int index); + int CountUntilWhitespace(Index index); /// /// Counts the amount of items, starting from a given , /// while a returns true. /// - /// - /// A function that takes the current index (relative to ), - /// which is incremented every item, and the item associated with it. Execution stops when - /// the predicate returns false or the index is out of bounds. - /// /// /// The index at which to start counting; all indexes will also be given to the /// as an offset that when added to the index of the position /// equal the actual index. /// + /// + /// A function that takes the current index (relative to ), + /// which is incremented every item, and the item associated with it. Execution stops when + /// the predicate returns false or the index is out of bounds. + /// /// The amount of items counted. - int CountWhile(Func predicate, int index); + int CountWhile(Index index, Func predicate); /// /// Returns the length of a line given its 0-based line number. @@ -112,7 +112,7 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable /// /// The 0-based index whose line is considered. /// The length of the line. - int GetLengthOfLineFromIndex(int index); + int GetLengthOfLineFromIndex(Index index); /// /// Given a 0-based line number, returns the matching line as an array of buffer items. @@ -127,29 +127,21 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable /// /// The index at which to determine what the current line is. /// The line, as an array of items. - ReadOnlySpan GetLineFromIndex(int index); + ReadOnlySpan GetLineFromIndex(Index index); /// /// Determines the 0-based line number of the line that contains the item located at . /// /// The index whose parent line's number is to be returned. /// The 0-based number of the line that contains item located at . - int GetLineNumberFromIndex(int index); + int GetLineNumberFromIndex(Index index); /// /// Converts an index into a location. /// /// The index. /// The location. - SourceLocation GetSourceLocation(int index); - - /// - /// Converts the buffer region into a span. - /// - /// The starting index. - /// The end index, which is included in the span. - /// The span. - SourceSpan GetSourceSpan(int startIndex, int endIndex); + (Index Index, int Line, int Column) GetLocationDetails(Index index); /// /// Slices a region of the buffer. @@ -157,37 +149,29 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable /// The index of the first item in the slice. /// The amount of items to slice starting at . /// A slice, as an array of items. - ReadOnlySpan Slice(int start, int length); + ReadOnlySpan Slice(Index start, int length); /// /// Slices a region of the buffer into a performant span. /// /// The index of the first item in the slice. /// The span serving as the destination for the slice. - bool TrySlice(int start, Span slice); + bool TrySlice(Index start, Span slice); /// /// Slices a region of the buffer into a performant span. /// - /// The span indicating what the slice is. + /// The range indicating what the slice is. /// The span serving as the destination for the slice. - bool TrySlice(SourceSpan sourceSpan, Span slice); - - /// - /// Tries to return the item at . - /// - /// The index of the character. - /// The item. - /// False if the buffer is out of bounds or an exception occured. - bool TryGetChar(int index, out char item); + bool TrySlice(Range range, Span slice); /// - /// Tries to return the item at the specified . + /// Tries to return the item at the specified . /// - /// The item's location. + /// The item's location. /// The item. /// False if the buffer is out of bounds or an exception occured. - bool TryGetChar(SourceLocation location, out char item); + bool TryGetChar(Index index, out char item); /// /// Given a 0-based line number, assigns the exact line to a result buffer (). @@ -205,5 +189,5 @@ public interface IBuffer : IDisposable, IEquatable, IEquatable /// The index at which to determine what the current line is. /// The destination span that will contain the line. /// True if successful. - bool TryGetLineFromIndex(int index, Span destination); + bool TryGetLineFromIndex(Index index, Span destination); } diff --git a/src/RSML.Toolchain.Abstractions/Sources/SourceLocation.cs b/src/RSML.Toolchain.Abstractions/Sources/SourceLocation.cs deleted file mode 100644 index cd94817..0000000 --- a/src/RSML.Toolchain.Abstractions/Sources/SourceLocation.cs +++ /dev/null @@ -1,125 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; - - -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Sources; - -/// -/// Specifies the location of an item in a or . -/// -/// The 0-based index. -/// The 0-based line number. -/// The 0-based column number (the index relative to the start of the line). -public readonly struct SourceLocation(int index, int line, int column) : IEquatable, IEquatable, IFormattable, - IComparable, IComparable -{ - /// - /// An empty source location. - /// - public static readonly SourceLocation Empty = new(0, 0, 0); - - /// - /// The 0-based line number, counting from the start of the source. - /// - public int Line => line; - - /// - /// The 0-based column number, which is the index of the item relative to the start of the line it is in. - /// - public int Column => column; - - /// - /// The absolute 0-based index of the item in the source. - /// - public int Index => index; - - /// - /// Compares the index of the location to another index. - /// - /// The index to compare against. - public int CompareTo(int other) => Index.CompareTo(other); - - /// - public int CompareTo(SourceLocation other) => throw new NotImplementedException(); - - /// - public override bool Equals( - [NotNullWhen(true)] - object? obj - ) => obj switch - { - SourceLocation location => Equals(location), - int idx => Index == idx, - _ => false - }; - - /// - /// Checks if two s are equal to each other. - /// - /// The other . - /// True if equals. - public bool Equals(SourceLocation other) => Index == other.Index && Line == other.Line && Column == other.Column; - - /// - /// Checks if two indexes are equal to each other. - /// - /// The other location's index. - /// True if equals. - public bool Equals(int other) => Index.Equals(other); - - /// - public override int GetHashCode() => unchecked(HashCode.Combine(Index, Line, Column)); - - /// - /// Returns a generic string representation of the current instance. - /// - /// The string representation. - public override string ToString() => $"SourceLocation(Index={Index}, Line={Line + 1}, Column={Column + 1})"; - - /// - /// Given a format, tries to return a string that uses said format as a basis for the representation. - /// If it fails, it defaults to . - /// - /// The format. Available formats are: CTOR (constructor-like string) and JSON (struct as JSON). - /// Unused. Don't bother assigning it anything. - /// The string representation. - public string ToString(string? format, IFormatProvider? formatProvider) => - format switch - { - "CTOR" or "I" or "INIT" or "NET" => $"new SourceLocation({Index}, {Line}, {Column})", - "JSON" => $$"""{ "index": {{Index}}, "line": {{Line + 1}}, "column": {{Column + 1}} }""", - _ => ToString() - }; - - /// - /// Checks if two s are equal to each other. - /// - /// True if equals. - public static bool operator ==(SourceLocation left, SourceLocation right) => left.Equals(right); - - /// - /// Checks if two s are different from each other. - /// - /// True if different. - public static bool operator !=(SourceLocation left, SourceLocation right) => !left.Equals(right); - - /// - /// Checks if is strictly less than . - /// - public static bool operator <(SourceLocation left, SourceLocation right) => left.Index < right.Index; - - /// - /// Checks if is strictly greater than . - /// - public static bool operator >(SourceLocation left, SourceLocation right) => left.Index > right.Index; - - /// - /// Checks if is greather than or equal to . - /// - public static bool operator >=(SourceLocation left, SourceLocation right) => left.Index >= right.Index; - - /// - /// Checks if is less than or equal to . - /// - public static bool operator <=(SourceLocation left, SourceLocation right) => left.Index <= right.Index; -} diff --git a/src/RSML.Toolchain.Abstractions/Sources/SourceSpan.cs b/src/RSML.Toolchain.Abstractions/Sources/SourceSpan.cs deleted file mode 100644 index 663241d..0000000 --- a/src/RSML.Toolchain.Abstractions/Sources/SourceSpan.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; - - -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Sources; - -/// -/// Represents a span taken from a source. -/// -public readonly struct SourceSpan : IFormattable, IEquatable -{ - /// - /// An empty span. - /// - public readonly static SourceSpan Empty = new(new(0, 0, 0), new(0, 0, 0)); - - /// - /// The start of the span. - /// - public readonly SourceLocation Start { get; } - - /// - /// The end of the span. - /// - public readonly SourceLocation End { get; } - - /// - /// The length of the span. - /// - public readonly int Length => End.Index - Start.Index; - - /// - /// The span is located in a single line. - /// - public bool IsSingleLine => Start.Line == End.Line; - - /// - /// Initializes a new span given a starting and an end indexes. - /// - /// The start index. - /// The end index. - /// The starting index is greater or equal to the end index. - public SourceSpan(SourceLocation start, SourceLocation end) - { - if (start.Index > end.Index) - throw new ArgumentException("The starting index must be less than the end index."); - - Start = start; - End = end; - } - - /// - public override bool Equals( - [NotNullWhen(true)] - object? obj - ) => obj is SourceSpan span && Equals(span); - - /// - /// Checks whether two s are equals. - /// - /// The span to check against - /// True if equals - public bool Equals(SourceSpan other) => Start.Equals(other.Start) && End.Equals(other.End); - - /// - /// Checks whether two s are equals. - /// - /// True if equals - public static bool operator ==(SourceSpan left, SourceSpan right) => left.Equals(right); - - /// - /// Checks whether two s are different from each other. - /// - /// True if different - public static bool operator !=(SourceSpan left, SourceSpan right) => left.Equals(right); - - /// - /// Returns a generic string representation of the current instance. - /// - /// The string representation. - public override string ToString() => $"SourceSpan(Start={Start}, End={End})"; - - /// - /// Given a format, tries to return a string that uses said format as a basis for the representation. - /// If it fails, it defaults to . - /// - /// The format. Available formats are: CTOR (constructor-like string) and JSON (struct as JSON). - /// Unused. Don't bother assigning it anything. - /// The string representation. - public string ToString(string? format, IFormatProvider? formatProvider) => - format switch - { - "CTOR" or "I" or "INIT" or "NET" => $"new SourceSpan({Start.ToString("ctor", null)}, {End.ToString("ctor", null)})", - "JSON" => $$"""{ "start": {{Start.ToString("JSON", null)}}, "end": {{End.ToString("JSON", null)}} }""", - _ => ToString() - }; - - /// - public override int GetHashCode() => unchecked(HashCode.Combine(Start, End)); -} diff --git a/src/RSML.Toolchain.Sources/GlobalSuppressions.cs b/src/RSML.Toolchain.Sources/GlobalSuppressions.cs index d9f1eaf..d76fa6b 100644 --- a/src/RSML.Toolchain.Sources/GlobalSuppressions.cs +++ b/src/RSML.Toolchain.Sources/GlobalSuppressions.cs @@ -5,6 +5,5 @@ using System.Diagnostics.CodeAnalysis; -[assembly: SuppressMessage("Design", "CA1043:Use Integral Or String Argument For Indexers", Justification = "Refers to a single location.", Scope = "member", Target = "~P:OceanApocalypse.RSML.Toolchain.Sources.ReadOnlyStringBuffer.Item(OceanApocalypse.RSML.Toolchain.Abstractions.Sources.SourceLocation)")] [assembly: SuppressMessage("CodeQuality", "IDE0079:Remove unnecessary suppression", Justification = "Not an unnecessary suppression.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Toolchain.Sources.ReadOnlyStringBuffer.#ctor(System.Byte*,System.Int32,System.Text.Encoding)")] [assembly: SuppressMessage("CodeQuality", "IDE0079:Remove unnecessary suppression", Justification = "Not an unnecessary suppression.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Toolchain.Sources.ReadOnlyStringBuffer.BuildCache")] diff --git a/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs b/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs index 6e4eb7d..f85e2aa 100644 --- a/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs +++ b/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs @@ -64,13 +64,13 @@ private int RawLineCount } /// - public char this[int index] => data[index]; + public ReadOnlySpan this[Range range] => data.AsSpan()[range]; /// - public char this[SourceLocation location] => this[location.Index]; + public char this[Index index] => data[index]; /// - public ReadOnlySpan this[SourceSpan span] => data.AsSpan().Slice(span.Start.Index, span.Length); + public char this[int index] => data[index]; /// /// Initializes a new @@ -141,31 +141,31 @@ public unsafe ReadOnlyStringBuffer(byte* contentPtr, int byteCount, Encoding? en /// ::: /// /// - public int CountUntilEndOfLine(int index, out bool isCrLf) + public int CountUntilEndOfLine(Index index, out bool isCrLf) { isCrLf = false; + int offset = index.GetOffset(Length); ThrowIfEmpty(); - index = NormalizeIndex(index); - ThrowIfOutOfRange(index, true); + ThrowIfOutOfRange(offset, true); - if (index == Length) + if (offset == Length) return 0; // consumed the entire buffer ComputeLineStarts(); - int lineSep = GetNextLineStartPosition(index, out _); + int lineSep = GetNextLineStartPosition(offset, out _); isCrLf = precededByCrLf.Contains(lineSep) && data[index] is not '\n'; // to us, CRLF is only when we're not standing on the LF if (isCrLf) lineSep--; // skip the extra line separator in the CRLF sequence - if (!(IsLastLine(index) && !data[^1].IsNewline())) // if we're not on the last line and it doesn't end with a newline then + if (!(IsLastLine(offset) && !data[^1].IsNewline())) // if we're not on the last line and it doesn't end with a newline then { lineSep--; } - return lineSep - index; + return lineSep - offset; } /// @@ -183,13 +183,13 @@ public int CountUntilEndOfLine(int index, out bool isCrLf) /// ::: /// /// - public int CountUntilNotWhitespace(int index) + public int CountUntilNotWhitespace(Index index) { ThrowIfEmpty(); - index = NormalizeIndex(index); - ThrowIfOutOfRange(index, true); + int offset = index.GetOffset(Length); + ThrowIfOutOfRange(offset, true); - if (index == Length) + if (offset == Length) return 0; // consumed the entire buffer var span = data.AsSpan(index); @@ -216,16 +216,16 @@ public int CountUntilNotWhitespace(int index) /// ::: /// /// - public int CountUntilWhitespace(int index) + public int CountUntilWhitespace(Index index) { ThrowIfEmpty(); - index = NormalizeIndex(index); - ThrowIfOutOfRange(index, true); + int offset = index.GetOffset(Length); + ThrowIfOutOfRange(offset, true); - if (index == Length) + if (offset == Length) return 0; // consumed the entire buffer - var span = data.AsSpan(index); + var span = data.AsSpan(offset); int count = 0; while (count < span.Length && !Char.IsWhiteSpace(span[count])) @@ -249,19 +249,19 @@ public int CountUntilWhitespace(int index) /// ::: /// /// - public int CountWhile(Func predicate, int index) + public int CountWhile(Index index, Func predicate) { if (predicate is null) throw new ArgumentNullException(nameof(predicate), "The object is null."); ThrowIfEmpty(); - index = NormalizeIndex(index); - ThrowIfOutOfRange(index, true); + int offset = index.GetOffset(Length); + ThrowIfOutOfRange(offset, true); - if (index == Length) + if (offset == Length) return 0; // consumed the entire buffer - var span = data.AsSpan(index); + var span = data.AsSpan(offset); int count = 0; while (count < span.Length && predicate(count, span[count])) @@ -312,7 +312,7 @@ public int GetLengthOfLine(int lineNumber) /// ::: /// /// - public int GetLengthOfLineFromIndex(int index) => GetLengthOfLine(GetLineNumberFromIndex(index)); + public int GetLengthOfLineFromIndex(Index index) => GetLengthOfLine(GetLineNumberFromIndex(index)); /// /// :::info[EOF Conventions] @@ -320,14 +320,14 @@ public int GetLengthOfLine(int lineNumber) /// ::: /// /// - public int GetLineNumberFromIndex(int index) + public int GetLineNumberFromIndex(Index index) { ThrowIfEmpty(); - index = NormalizeIndex(index); - ThrowIfOutOfRange(index, true); + int offset = index.GetOffset(Length); + ThrowIfOutOfRange(offset, true); ComputeLineStarts(); - int lineSepIndex = GetPreviousOrCurrentLineStartPositionInLineStartList(index); + int lineSepIndex = GetPreviousOrCurrentLineStartPositionInLineStartList(offset); return lineSepIndex; } @@ -356,39 +356,26 @@ public ReadOnlySpan GetLine(int lineNumber) } /// - public ReadOnlySpan GetLineFromIndex(int index) => GetLine(GetLineNumberFromIndex(index)); + public ReadOnlySpan GetLineFromIndex(Index index) => GetLine(GetLineNumberFromIndex(index)); /// /// :::warning[EOF Conventions] /// Unlike with other methods, this one /// does not follow EOF conventions and, because of that, does not accept the /// EOF index (index at ), because it is not - /// considered a location. + /// considered part of any slice. /// ::: /// /// - public SourceLocation GetSourceLocation(int index) + public ReadOnlySpan Slice(Index start, int length) { - ThrowIfEmpty(); - index = NormalizeIndex(index); - ThrowIfOutOfRange(index); - - if (index == 0) // best "best" case = triple zero - return SourceLocation.Empty; - - ComputeLineStarts(); - int lineNumber = GetLineNumberFromIndex(index); - - return new(index, lineNumber, index - lineStarts[lineNumber]); - } + if (length < 0) + throw new ArgumentOutOfRangeException(nameof(length), "The slice length must be positive."); - /// - public SourceSpan GetSourceSpan(int startIndex, int endIndex) - { - var start = GetSourceLocation(startIndex); - var end = GetSourceLocation(endIndex); + int offset = start.GetOffset(Length); + ThrowIfOutOfRange(offset, true, nameof(start)); - return new(start, end); + return data.AsSpan(offset, length); } /// @@ -400,15 +387,15 @@ public SourceSpan GetSourceSpan(int startIndex, int endIndex) /// ::: /// /// - public ReadOnlySpan Slice(int start, int length) + public ReadOnlySpan Slice(Range range) { - if (length < 0) - throw new ArgumentOutOfRangeException(nameof(length), "The slice length must be positive."); + int startOffset = range.Start.GetOffset(Length); + int endOffset = range.End.GetOffset(Length); - start = NormalizeIndex(start); - ThrowIfOutOfRange(start, true, nameof(start)); + ThrowIfOutOfRange(startOffset, true, nameof(range)); + ThrowIfOutOfRange(endOffset, true, nameof(range)); - return data.AsSpan(start, length); + return data.AsSpan()[range]; } /// @@ -420,18 +407,10 @@ public ReadOnlySpan Slice(int start, int length) /// ::: /// /// - public bool TrySlice(int start, Span slice) => data.AsSpan(NormalizeIndex(start), slice.Length).TryCopyTo(slice); + public bool TrySlice(Index start, Span slice) => data.AsSpan(start.GetOffset(Length), slice.Length).TryCopyTo(slice); - /// - /// :::warning[EOF Conventions] - /// Unlike with other methods, this one - /// does not follow EOF conventions and, because of that, does not accept the - /// EOF index (index at ), because it is not - /// considered part of any slice. - /// ::: - /// /// - public bool TrySlice(SourceSpan sourceSpan, Span slice) => data.AsSpan(sourceSpan.Start.Index, sourceSpan.Length).TryCopyTo(slice); + public bool TrySlice(Range range, Span slice) => data.AsSpan()[range].TryCopyTo(slice); /// /// :::info[EOF Conventions] @@ -441,16 +420,11 @@ public ReadOnlySpan Slice(int start, int length) /// ::: /// /// - public bool TryGetChar(int index, out char item) + public bool TryGetChar(Index index, out char item) { item = '\0'; // default - if (IsEmpty) - return false; - - index = NormalizeIndex(index); - - if (IsOutOfRange(index)) + if (IsEmpty || IsOutOfRange(index.IsFromEnd ? Length + (-index.Value) : index.Value)) return false; item = data[index]; @@ -458,16 +432,6 @@ public bool TryGetChar(int index, out char item) return true; } - /// - /// :::info[EOF Conventions] - /// This method follows the EOF convention where the EOF character - /// is 0 ('\0') and the return value is false, due to EOF - /// not being an actual buffer location. - /// ::: - /// - /// - public bool TryGetChar(SourceLocation location, out char item) => TryGetChar(location.Index, out item); - /// /// :::info[EOF Conventions] /// This method follows EOF conventions. @@ -503,17 +467,38 @@ public bool TryGetLine(int lineNumber, Span destination) /// ::: /// /// - public bool TryGetLineFromIndex(int index, Span destination) + public bool TryGetLineFromIndex(Index index, Span destination) { - index = NormalizeIndex(index); - - if (IsEmpty || IsOutOfRange(index, followEofConvention: true)) // avoids panic from GetLineNumberFromIndex + if (IsEmpty || IsOutOfRange(index.IsFromEnd ? Length + (-index.Value) : index.Value, followEofConvention: true)) // avoids panic from GetLineNumberFromIndex return false; var lineNumber = GetLineNumberFromIndex(index); return TryGetLine(lineNumber, destination); } + /// + /// :::warning[EOF Conventions] + /// Unlike with other methods, this one + /// does not follow EOF conventions and, because of that, does not accept the + /// EOF index (index at ), because it is not + /// considered a location. + /// ::: + /// + /// + public (Index Index, int Line, int Column) GetLocationDetails(Index index) + { + ThrowIfEmpty(); + ThrowIfOutOfRange(index.IsFromEnd ? Length + (-index.Value) : index.Value); + + if (index.Value == 0) // best "best" case = triple zero + return (0, 0, 0); + + ComputeLineStarts(); + int lineNumber = GetLineNumberFromIndex(index); + + return new(index, lineNumber, index.GetOffset(Length) - lineStarts[lineNumber]); + } + /// public void Dispose() { diff --git a/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs b/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs index 3dbabe8..2080022 100644 --- a/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs +++ b/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs @@ -81,7 +81,7 @@ public class ReadOnlyStringBufferTests public void CountUntilEndOfLine(string data, int index, int expectedCount, bool expectedCrLf) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountUntilEndOfLine(index, out bool actualCrLf)); + Assert.Equal(expectedCount, buffer.CountUntilEndOfLine(index < 0 ? new(-index, true) : (Index)index, out bool actualCrLf)); Assert.Equal(expectedCrLf, actualCrLf); } @@ -98,7 +98,7 @@ public void CountUntilEndOfLine_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); bool isCrLf = true; - Assert.Throws(() => buffer.CountUntilEndOfLine(index, out isCrLf)); + Assert.Throws(() => buffer.CountUntilEndOfLine(index < 0 ? new(-index, true) : (Index)index, out isCrLf)); Assert.False(isCrLf); } @@ -114,7 +114,7 @@ public void CountUntilEndOfLine_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); bool isCrLf = true; - Assert.Throws(() => buffer.CountUntilEndOfLine(index, out isCrLf)); + Assert.Throws(() => buffer.CountUntilEndOfLine(index < 0 ? new(-index, true) : (Index)index, out isCrLf)); Assert.False(isCrLf); } @@ -156,7 +156,7 @@ public void CountUntilEndOfLine_FailsIfOutOfRange(string data, int index) public void CountUntilNotWhitespace(string data, int index, int expectedCount) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountUntilNotWhitespace(index)); + Assert.Equal(expectedCount, buffer.CountUntilNotWhitespace(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -170,7 +170,7 @@ public void CountUntilNotWhitespace(string data, int index, int expectedCount) public void CountUntilNotWhitespace_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.CountUntilNotWhitespace(index)); + Assert.Throws(() => buffer.CountUntilNotWhitespace(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -182,7 +182,7 @@ public void CountUntilNotWhitespace_FailsIfEmpty(int index) public void CountUntilNotWhitespace_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.CountUntilNotWhitespace(index)); + Assert.Throws(() => buffer.CountUntilNotWhitespace(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -223,7 +223,7 @@ public void CountUntilNotWhitespace_FailsIfOutOfRange(string data, int index) public void CountUntilWhitespace(string data, int index, int expectedCount) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountUntilWhitespace(index)); + Assert.Equal(expectedCount, buffer.CountUntilWhitespace(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -237,7 +237,7 @@ public void CountUntilWhitespace(string data, int index, int expectedCount) public void CountUntilWhitespace_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.CountUntilWhitespace(index)); + Assert.Throws(() => buffer.CountUntilWhitespace(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -249,7 +249,7 @@ public void CountUntilWhitespace_FailsIfEmpty(int index) public void CountUntilWhitespace_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.CountUntilWhitespace(index)); + Assert.Throws(() => buffer.CountUntilWhitespace(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -262,7 +262,7 @@ public void CountUntilWhitespace_FailsIfOutOfRange(string data, int index) public void CountWhile_SameAsLengthIfAlwaysTrue(string data) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(data.Length, buffer.CountWhile((_, _) => true, 0)); + Assert.Equal(data.Length, buffer.CountWhile(0, (_, _) => true)); } [Theory] @@ -326,7 +326,7 @@ public void CountWhile_SameAsLengthIfAlwaysTrue(string data) public void CountWhile_CountsWhileNotLowercaseR(string data, int index, int expectedCount) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountWhile((_, c) => c != 'r', index)); + Assert.Equal(expectedCount, buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, c) => c != 'r')); } [Theory] @@ -373,7 +373,7 @@ public void CountWhile_CountsWhileNotLowercaseR(string data, int index, int expe public void CountWhile_CountsWhileUppercaseOrWhitespace(string data, int index, int expectedCount) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountWhile((_, c) => Char.IsWhiteSpace(c) || c is '\r' or '\n' or '\u2028' or '\u2029' || Char.IsUpper(c), index)); + Assert.Equal(expectedCount, buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, c) => Char.IsWhiteSpace(c) || c is '\r' or '\n' or '\u2028' or '\u2029' || Char.IsUpper(c))); } [Theory] @@ -387,7 +387,7 @@ public void CountWhile_CountsWhileUppercaseOrWhitespace(string data, int index, public void CountWhile_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.CountWhile((_, _) => true, index)); + Assert.Throws(() => buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, _) => true)); } [Theory] @@ -399,7 +399,7 @@ public void CountWhile_FailsIfEmpty(int index) public void CountWhile_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.CountWhile((_, _) => true, index)); + Assert.Throws(() => buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, _) => true)); } [Theory] @@ -526,10 +526,10 @@ public void GetLengthOfLine_FailsIfOutOfRange(string data, int lineNumber) [InlineData(TestString02, -1, 1)] // Dot/point in "\r\n\r\n." [InlineData(TestString02, 33, 0)] // End of file #endregion - public void GetLengthOfLineFromIndex(string data, int lineNumber, int expectedLength) + public void GetLengthOfLineFromIndex(string data, int index, int expectedLength) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedLength, buffer.GetLengthOfLineFromIndex(lineNumber)); + Assert.Equal(expectedLength, buffer.GetLengthOfLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -540,10 +540,10 @@ public void GetLengthOfLineFromIndex(string data, int lineNumber, int expectedLe [InlineData(136)] [InlineData(-4)] #endregion - public void GetLengthOfLineFromIndex_FailsIfEmpty(int lineNumber) + public void GetLengthOfLineFromIndex_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLengthOfLineFromIndex(lineNumber)); + Assert.Throws(() => buffer.GetLengthOfLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -552,10 +552,10 @@ public void GetLengthOfLineFromIndex_FailsIfEmpty(int lineNumber) [InlineData(TestString01, 35)] [InlineData(TestString01, -35)] #endregion - public void GetLengthOfLineFromIndex_FailsIfOutOfRange(string data, int lineNumber) + public void GetLengthOfLineFromIndex_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLengthOfLineFromIndex(lineNumber)); + Assert.Throws(() => buffer.GetLengthOfLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -664,7 +664,7 @@ public void GetLine_FailsIfOutOfRange(string data, int lineNumber) public void GetLineFromIndex(string data, int index, string expectedLine) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedLine, buffer.GetLineFromIndex(index)); + Assert.Equal(expectedLine, buffer.GetLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -673,10 +673,10 @@ public void GetLineFromIndex(string data, int index, string expectedLine) [InlineData(TestString01, 35)] [InlineData(TestString01, -35)] #endregion - public void GetLineFromIndex_FailsIfOutOfRange(string data, int lineNumber) + public void GetLineFromIndex_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLineFromIndex(lineNumber)); + Assert.Throws(() => buffer.GetLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -687,10 +687,10 @@ public void GetLineFromIndex_FailsIfOutOfRange(string data, int lineNumber) [InlineData(136)] [InlineData(-4)] #endregion - public void GetLineFromIndex_FailsIfEmpty(int lineNumber) + public void GetLineFromIndex_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLineFromIndex(lineNumber)); + Assert.Throws(() => buffer.GetLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -727,11 +727,11 @@ public void GetLineFromIndex_FailsIfEmpty(int lineNumber) [InlineData(TestString02, -1, 6)] // Dot/point in "\r\n\r\n." [InlineData(TestString02, 33, 7)] // EOF #endregion - public void GetLineNumberFromIndex(string data, int index, int expectedLineCount) + public void GetLineNumberFromIndex(string data, int index, int expectedLineNumber) { var buffer = new ReadOnlyStringBuffer(data); buffer.BuildCache(); - Assert.Equal(expectedLineCount, buffer.GetLineNumberFromIndex(index)); + Assert.Equal(expectedLineNumber, buffer.GetLineNumberFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -740,10 +740,10 @@ public void GetLineNumberFromIndex(string data, int index, int expectedLineCount [InlineData(TestString01, 35)] [InlineData(TestString01, -35)] #endregion - public void GetLineNumberFromIndex_FailsIfOutOfRange(string data, int lineNumber) + public void GetLineNumberFromIndex_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLineNumberFromIndex(lineNumber)); + Assert.Throws(() => buffer.GetLineNumberFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -754,50 +754,49 @@ public void GetLineNumberFromIndex_FailsIfOutOfRange(string data, int lineNumber [InlineData(136)] [InlineData(-4)] #endregion - public void GetLineNumberFromIndex_FailsIfEmpty(int lineNumber) + public void GetLineNumberFromIndex_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLineNumberFromIndex(lineNumber)); + Assert.Throws(() => buffer.GetLineNumberFromIndex(index < 0 ? new(-index, true) : (Index)index)); } [Theory] #region String ends with newline - [InlineData(TestString01, 0, 0, 0, 0)] // H in "Hey" - [InlineData(TestString01, 3, 3, 0, 3)] // CR in "Hey\r\n" - [InlineData(TestString01, 4, 4, 0, 4)] // LF in "Hey\r\n" - [InlineData(TestString01, 5, 5, 1, 0)] // T in "This" - [InlineData(TestString01, 6, 6, 1, 1)] // h in "This" - [InlineData(TestString01, 13, 13, 3, 0)] // A in "A Test" - [InlineData(TestString01, 15, 15, 3, 2)] // T in "Test" - [InlineData(TestString01, 22, 22, 4, 1)] // M in "Method" - [InlineData(TestString01, 28, 28, 4, 7)] // First CR in "\r\n\r\n." - [InlineData(TestString01, -5, 29, 4, 8)] // First LF in "\r\n\r\n." - [InlineData(TestString01, 30, 30, 5, 0)] // Second CR in "\r\n\r\n." - [InlineData(TestString01, -3, 31, 5, 1)] // Second LF in "\r\n\r\n." - [InlineData(TestString01, 32, 32, 6, 0)] // Dot/point in "\r\n\r\n." - [InlineData(TestString01, 33, 33, 6, 1)] // U2028 in ".\u2028" + [InlineData(TestString01, 0, 0, 0)] // H in "Hey" + [InlineData(TestString01, 3, 0, 3)] // CR in "Hey\r\n" + [InlineData(TestString01, 4, 0, 4)] // LF in "Hey\r\n" + [InlineData(TestString01, 5, 1, 0)] // T in "This" + [InlineData(TestString01, 6, 1, 1)] // h in "This" + [InlineData(TestString01, 13, 3, 0)] // A in "A Test" + [InlineData(TestString01, 15, 3, 2)] // T in "Test" + [InlineData(TestString01, 22, 4, 1)] // M in "Method" + [InlineData(TestString01, 28, 4, 7)] // First CR in "\r\n\r\n." + [InlineData(TestString01, -5, 4, 8)] // First LF in "\r\n\r\n." + [InlineData(TestString01, 30, 5, 0)] // Second CR in "\r\n\r\n." + [InlineData(TestString01, -3, 5, 1)] // Second LF in "\r\n\r\n." + [InlineData(TestString01, 32, 6, 0)] // Dot/point in "\r\n\r\n." + [InlineData(TestString01, 33, 6, 1)] // U2028 in ".\u2028" #endregion #region String ends without newline - [InlineData(TestString02, 0, 0, 0, 0)] // H in "Hey" - [InlineData(TestString02, 3, 3, 0, 3)] // CR in "Hey\r\n" - [InlineData(TestString02, 4, 4, 0, 4)] // LF in "Hey\r\n" - [InlineData(TestString02, 5, 5, 1, 0)] // T in "This" - [InlineData(TestString02, 6, 6, 1, 1)] // h in "This" - [InlineData(TestString02, 13, 13, 3, 0)] // A in "A Test" - [InlineData(TestString02, 15, 15, 3, 2)] // T in "Test" - [InlineData(TestString02, 22, 22, 4, 1)] // M in "Method" - [InlineData(TestString02, 28, 28, 4, 7)] // First CR in "\r\n\r\n." - [InlineData(TestString02, 29, 29, 4, 8)] // First LF in "\r\n\r\n." - [InlineData(TestString02, 30, 30, 5, 0)] // Second CR in "\r\n\r\n." - [InlineData(TestString02, -2, 31, 5, 1)] // Second LF in "\r\n\r\n." - [InlineData(TestString02, -1, 32, 6, 0)] // Dot/point in "\r\n\r\n." - #endregion - public void GetSourceLocation(string data, int index, int expectedIndex, int expectedLine, int expectedColumn) + [InlineData(TestString02, 0, 0, 0)] // H in "Hey" + [InlineData(TestString02, 3, 0, 3)] // CR in "Hey\r\n" + [InlineData(TestString02, 4, 0, 4)] // LF in "Hey\r\n" + [InlineData(TestString02, 5, 1, 0)] // T in "This" + [InlineData(TestString02, 6, 1, 1)] // h in "This" + [InlineData(TestString02, 13, 3, 0)] // A in "A Test" + [InlineData(TestString02, 15, 3, 2)] // T in "Test" + [InlineData(TestString02, 22, 4, 1)] // M in "Method" + [InlineData(TestString02, 28, 4, 7)] // First CR in "\r\n\r\n." + [InlineData(TestString02, 29, 4, 8)] // First LF in "\r\n\r\n." + [InlineData(TestString02, 30, 5, 0)] // Second CR in "\r\n\r\n." + [InlineData(TestString02, -2, 5, 1)] // Second LF in "\r\n\r\n." + [InlineData(TestString02, -1, 6, 0)] // Dot/point in "\r\n\r\n." + #endregion + public void GetLocationDetails(string data, int index, int expectedLine, int expectedColumn) { var buffer = new ReadOnlyStringBuffer(data); - var location = buffer.GetSourceLocation(index); + var location = buffer.GetLocationDetails(index < 0 ? new(-index, true) : (Index)index); - Assert.Equal(expectedIndex, location.Index); Assert.Equal(expectedLine, location.Line); Assert.Equal(expectedColumn, location.Column); } @@ -809,10 +808,10 @@ public void GetSourceLocation(string data, int index, int expectedIndex, int exp [InlineData(TestString01, 35)] [InlineData(TestString01, -35)] #endregion - public void GetSourceLocation_FailsIfOutOfRange(string data, int lineNumber) + public void GetLocationDetails_FailsIfOutOfRange(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetSourceLocation(lineNumber)); + Assert.Throws(() => buffer.GetLocationDetails(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -823,45 +822,10 @@ public void GetSourceLocation_FailsIfOutOfRange(string data, int lineNumber) [InlineData(136)] [InlineData(-4)] #endregion - public void GetSourceLocation_FailsIfEmpty(int lineNumber) + public void GetLocationDetails_FailsIfEmpty(int index) { var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetSourceLocation(lineNumber)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 3, new int[] { 0, 0, 0, 3, 0, 3 })] - [InlineData(TestString01, 3, 5, new int[] { 3, 0, 3, 5, 1, 0 })] - [InlineData(TestString01, 4, 33, new int[] { 4, 0, 4, 33, 6, 1 })] - [InlineData(TestString01, 5, 6, new int[] { 5, 1, 0, 6, 1, 1 })] - [InlineData(TestString01, 6, 15, new int[] { 6, 1, 1, 15, 3, 2 })] - [InlineData(TestString01, 13, 16, new int[] { 13, 3, 0, 16, 3, 3 })] - [InlineData(TestString01, 15, 22, new int[] { 15, 3, 2, 22, 4, 1 })] - [InlineData(TestString01, 28, -5, new int[] { 28, 4, 7, 29, 4, 8 })] - [InlineData(TestString01, 30, 32, new int[] { 30, 5, 0, 32, 6, 0 })] - [InlineData(TestString01, -3, 33, new int[] { 31, 5, 1, 33, 6, 1 })] - #endregion - #region String ends with newline - [InlineData(TestString02, 0, 3, new int[] { 0, 0, 0, 3, 0, 3 })] - [InlineData(TestString02, 3, 5, new int[] { 3, 0, 3, 5, 1, 0 })] - [InlineData(TestString02, 5, 6, new int[] { 5, 1, 0, 6, 1, 1 })] - [InlineData(TestString02, 6, 15, new int[] { 6, 1, 1, 15, 3, 2 })] - [InlineData(TestString02, 13, 16, new int[] { 13, 3, 0, 16, 3, 3 })] - [InlineData(TestString02, 15, 22, new int[] { 15, 3, 2, 22, 4, 1 })] - #endregion - public void GetSourceSpan(string data, int startIndex, int endIndex, int[] expectations) - { - var buffer = new ReadOnlyStringBuffer(data); - var span = buffer.GetSourceSpan(startIndex, endIndex); - - Assert.Equal(expectations[0], span.Start.Index); - Assert.Equal(expectations[1], span.Start.Line); - Assert.Equal(expectations[2], span.Start.Column); - - Assert.Equal(expectations[3], span.End.Index); - Assert.Equal(expectations[4], span.End.Line); - Assert.Equal(expectations[5], span.End.Column); + Assert.Throws(() => buffer.GetLocationDetails(index < 0 ? new(-index, true) : (Index)index)); } [Theory] @@ -920,7 +884,7 @@ public void IndexAccessor(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); Assert.Equal(data[index], buffer[index]); - Assert.Equal(data[index], buffer[new SourceLocation(index, 0, 0)]); + Assert.Equal(data[index], buffer[index < 0 ? new Index(-index, true) : (Index)index]); } [Theory] @@ -1010,10 +974,10 @@ public void Constructor_ReadOnlySpan(string data) [InlineData(TestString01, 30, 3, "\r\n.")] [InlineData(TestString01, -3, 3, "\n.\u2028")] #endregion - public void Slice_CharArray(string data, int index, int length, string expectedSlice) + public void Slice_CharSpan(string data, int index, int length, string expectedSlice) { var buffer = new ReadOnlyStringBuffer(data); - var slice = buffer.Slice(index, length); + var slice = buffer.Slice(index < 0 ? new(-index, true) : (Index)index, length); Assert.Equal(expectedSlice, new string(slice)); } @@ -1029,12 +993,12 @@ public void Slice_CharArray(string data, int index, int length, string expectedS [InlineData(TestString01, 30, "\r\n.")] [InlineData(TestString01, -3, "\n.\u2028")] #endregion - public void Slice_CharSpan(string data, int index, string expectedSlice) + public void TrySlice_CharSpan(string data, int index, string expectedSlice) { Span span = stackalloc char[expectedSlice.Length]; var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TrySlice(index, span)); + Assert.True(buffer.TrySlice(index < 0 ? new(-index, true) : (Index)index, span)); Assert.Equal(expectedSlice, span); } @@ -1050,21 +1014,21 @@ public void Slice_CharSpan(string data, int index, string expectedSlice) [InlineData(TestString01, 30, 33, "\r\n.")] [InlineData(TestString01, 31, 34, "\n.\u2028")] #endregion - public void Slice_SourceSpan(string data, int startIndex, int endIndex, string expectedSlice) + public void Slice_Range(string data, int startIndex, int endIndex, string expectedSlice) { Span span = stackalloc char[expectedSlice.Length]; var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TrySlice(new SourceSpan(new(startIndex, 0, 0), new(endIndex, 0, 0)), span)); + Assert.True(buffer.TrySlice(startIndex..endIndex, span)); Assert.Equal(expectedSlice, span); } [Fact] - public void Slice_SourceSpan_FailsIfSpanTooSmall() + public void Slice_Range_FailsIfSpanTooSmall() { var buffer = new ReadOnlyStringBuffer(TestString05); - Assert.False(buffer.TrySlice(new SourceSpan(new(0, 0, 0), new(7, 0, 0)), stackalloc char[3])); + Assert.False(buffer.TrySlice(0..7, stackalloc char[3])); } [Fact] @@ -1129,7 +1093,7 @@ public void ToString_SameAsInputData() public void TryGetChar(string data, int index) { var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TryGetChar(index, out char item)); + Assert.True(buffer.TryGetChar(index < 0 ? new(-index, true) : (Index)index, out char item)); Assert.Equal(data[index], item); } @@ -1265,7 +1229,7 @@ public void TryGetLineFromIndex(string data, int index, string expectedLine) Span span = stackalloc char[expectedLine.Length]; var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TryGetLineFromIndex(index, span)); + Assert.True(buffer.TryGetLineFromIndex(index < 0 ? new(-index, true) : (Index)index, span)); Assert.Equal(expectedLine, span.ToString()); } @@ -1290,7 +1254,7 @@ public void TryGetLineFromIndex_FailsIfOutOfRange() Assert.False(buffer.TryGetLineFromIndex(12, span)); Assert.Equal(emptySequence, span); - Assert.False(buffer.TryGetLineFromIndex(-12, span)); + Assert.False(buffer.TryGetLineFromIndex(^12, span)); Assert.Equal(emptySequence, span); } From 891d6a9ed095163dc6f4bba36cbbc8d761a5ea29 Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 9 Aug 2026 23:41:04 +0100 Subject: [PATCH 04/11] refactor(solution): Organize solution properties better Essentially, splitting the previously huge `Directory.Build.props`. --- .editorconfig | 125 +----------------- Directory.Build.props | 66 ++------- RedSeaModernLanguage.slnx | 22 ++- benchmarks/RSML.Benchmarks.csproj | 4 +- props/Analyzers.props | 15 +++ props/Common.props | 36 +++++ props/Defaults.props | 9 ++ props/NuGet.props | 23 ++++ props/README.md | 47 +++++++ props/Versioning.props | 25 ++++ .../RSML.Abstractions.Diagnostics.csproj | 18 +++ .../RSML.Abstractions.Panic.csproj | 14 ++ .../RSML.Abstractions.csproj | 18 +++ .../RSML.Language.Lexing.csproj | 40 +----- .../RSML.Language.Parsing.csproj | 38 +----- src/RSML.Native/RSML.Native.csproj | 18 --- .../RSML.Toolchain.Abstractions.csproj | 50 ------- .../RSML.Toolchain.Analysis.csproj | 36 ----- .../RSML.Toolchain.Execution.csproj | 36 ----- .../Interpreter.cs | 2 +- ...L.Toolchain.Extensibility.Execution.csproj | 41 +----- ...RSML.Toolchain.Extensibility.Lexing.csproj | 36 ----- ...SML.Toolchain.Extensibility.Parsing.csproj | 37 ------ .../RSML.Toolchain.Sources.csproj | 39 +----- .../ReadOnlyStringBuffer.cs | 8 +- src/RSML/RSML.csproj | 41 ------ .../RSML.InternalTests.csproj | 3 - .../RSML.NativeTests/RSML.NativeTests.csproj | 6 +- tests/RSML.Tests/RSML.Tests.csproj | 6 +- .../Sources/ReadOnlyStringBufferTests.cs | 3 +- 30 files changed, 252 insertions(+), 610 deletions(-) create mode 100644 props/Analyzers.props create mode 100644 props/Common.props create mode 100644 props/Defaults.props create mode 100644 props/NuGet.props create mode 100644 props/README.md create mode 100644 props/Versioning.props create mode 100644 src/RSML.Abstractions.Diagnostics/RSML.Abstractions.Diagnostics.csproj create mode 100644 src/RSML.Abstractions.Panic/RSML.Abstractions.Panic.csproj create mode 100644 src/RSML.Abstractions/RSML.Abstractions.csproj delete mode 100644 src/RSML.Toolchain.Abstractions/RSML.Toolchain.Abstractions.csproj diff --git a/.editorconfig b/.editorconfig index 4943eb4..0959e51 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,62 +1,34 @@ -# Remove the line below if you want to inherit .editorconfig settings from higher directories root = true -# C# files [*.{cs,csx}] - -#### Core EditorConfig Options #### - -# Indentation and spacing indent_size = 4 indent_style = tab tab_width = 4 - -# New line preferences end_of_line = lf insert_final_newline = true - -#### .NET Code Actions #### - -# Type members dotnet_hide_advanced_members = false dotnet_member_insertion_location = with_other_members_of_the_same_kind dotnet_property_generation_behavior = prefer_throwing_properties - -# Symbol search dotnet_search_reference_assemblies = true - -#### .NET Coding Conventions #### - -# Organize usings dotnet_separate_import_directive_groups = true dotnet_sort_system_directives_first = true file_header_template = unset - -# this. and Me. preferences dotnet_style_qualification_for_event = false:warning dotnet_style_qualification_for_field = false dotnet_style_qualification_for_method = false:warning dotnet_style_qualification_for_property = false:warning - -# Language keywords vs BCL types preferences dotnet_style_predefined_type_for_locals_parameters_members = true:warning dotnet_style_predefined_type_for_member_access = false:warning - -# Parentheses preferences dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity dotnet_style_parentheses_in_other_binary_operators = always_for_clarity dotnet_style_parentheses_in_other_operators = never_if_unnecessary:suggestion dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity - -# Modifier preferences dotnet_style_require_accessibility_modifiers = for_non_interface_members - -# Expression-level preferences dotnet_prefer_system_hash_code = true:warning dotnet_style_coalesce_expression = true:warning dotnet_style_collection_initializer = true:warning dotnet_style_explicit_tuple_names = true:warning -dotnet_style_namespace_match_folder = true +dotnet_style_namespace_match_folder = true:warning dotnet_style_null_propagation = true:warning dotnet_style_object_initializer = true:warning dotnet_style_operator_placement_when_wrapping = beginning_of_line @@ -72,28 +44,14 @@ dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning dotnet_style_prefer_non_hidden_explicit_cast_in_source = true dotnet_style_prefer_simplified_boolean_expressions = true:warning dotnet_style_prefer_simplified_interpolation = true - -# Field preferences dotnet_style_readonly_field = true - -# Parameter preferences dotnet_code_quality_unused_parameters = all:warning - -# Suppression preferences dotnet_remove_unnecessary_suppression_exclusions = none - -# New line preferences dotnet_style_allow_multiple_blank_lines_experimental = true dotnet_style_allow_statement_immediately_after_block_experimental = false:warning - -#### C# Coding Conventions #### - -# var preferences csharp_style_var_elsewhere = true csharp_style_var_for_built_in_types = false csharp_style_var_when_type_is_apparent = false - -# Expression-bodied members csharp_style_expression_bodied_accessors = true:warning csharp_style_expression_bodied_constructors = true:warning csharp_style_expression_bodied_indexers = true:warning @@ -102,26 +60,18 @@ csharp_style_expression_bodied_local_functions = true:warning csharp_style_expression_bodied_methods = true:warning csharp_style_expression_bodied_operators = true:warning csharp_style_expression_bodied_properties = true:warning - -# Pattern matching preferences csharp_style_pattern_matching_over_as_with_null_check = true:warning csharp_style_pattern_matching_over_is_with_cast_check = true:warning csharp_style_prefer_extended_property_pattern = true csharp_style_prefer_not_pattern = true:warning csharp_style_prefer_pattern_matching = true:warning csharp_style_prefer_switch_expression = true:warning - -# Null-checking preferences csharp_style_conditional_delegate_call = true:warning - -# Modifier preferences csharp_prefer_static_anonymous_function = true csharp_prefer_static_local_function = true csharp_preferred_modifier_order = public,private,protected,internal,file,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,required,volatile,async csharp_style_prefer_readonly_struct = true csharp_style_prefer_readonly_struct_member = true - -# Code-block preferences csharp_prefer_braces = when_multiline:warning csharp_prefer_simple_using_statement = true:warning csharp_prefer_system_threading_lock = true @@ -130,8 +80,6 @@ csharp_style_prefer_method_group_conversion = true:warning csharp_style_prefer_primary_constructors = true csharp_style_prefer_simple_property_accessors = true csharp_style_prefer_top_level_statements = false:error - -# Expression-level preferences csharp_prefer_simple_default_expression = true:warning csharp_style_deconstructed_variable_declaration = true csharp_style_implicit_object_creation_when_type_is_apparent = true:error @@ -147,20 +95,12 @@ csharp_style_prefer_utf8_string_literals = true csharp_style_throw_expression = true:warning csharp_style_unused_value_assignment_preference = discard_variable csharp_style_unused_value_expression_statement_preference = discard_variable - -# 'using' directive preferences csharp_using_directive_placement = outside_namespace:error - -# New line preferences csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true csharp_style_allow_blank_line_after_token_in_arrow_expression_clause_experimental = true csharp_style_allow_blank_line_after_token_in_conditional_expression_experimental = false:warning csharp_style_allow_blank_lines_between_consecutive_braces_experimental = false:warning csharp_style_allow_embedded_statements_on_same_line_experimental = false:warning - -#### C# Formatting Rules #### - -# New line preferences csharp_new_line_before_catch = true csharp_new_line_before_else = true csharp_new_line_before_finally = true @@ -168,16 +108,12 @@ csharp_new_line_before_members_in_anonymous_types = true csharp_new_line_before_members_in_object_initializers = true csharp_new_line_before_open_brace = all csharp_new_line_between_query_expression_clauses = true - -# Indentation preferences csharp_indent_block_contents = true csharp_indent_braces = false csharp_indent_case_contents = true csharp_indent_case_contents_when_block = true csharp_indent_labels = one_less_than_current csharp_indent_switch_labels = true - -# Space preferences csharp_space_after_cast = false csharp_space_after_colon_in_inheritance_clause = true csharp_space_after_comma = true @@ -200,128 +136,71 @@ csharp_space_between_method_declaration_name_and_open_parenthesis = false csharp_space_between_method_declaration_parameter_list_parentheses = false csharp_space_between_parentheses = false csharp_space_between_square_brackets = false - -# Wrapping preferences csharp_preserve_single_line_blocks = true csharp_preserve_single_line_statements = false - -#### Naming styles #### - -# Naming rules - dotnet_naming_rule.interfaces_should_be_begins_with_i.severity = error dotnet_naming_rule.interfaces_should_be_begins_with_i.symbols = interfaces dotnet_naming_rule.interfaces_should_be_begins_with_i.style = begins_with_i - dotnet_naming_rule.classes_should_be_pascalcase.severity = error dotnet_naming_rule.classes_should_be_pascalcase.symbols = classes dotnet_naming_rule.classes_should_be_pascalcase.style = pascalcase - dotnet_naming_rule.structures_and_enums_should_be_pascalcase.severity = error dotnet_naming_rule.structures_and_enums_should_be_pascalcase.symbols = structures_and_enums dotnet_naming_rule.structures_and_enums_should_be_pascalcase.style = pascalcase - dotnet_naming_rule.async_methods_should_be_ends_with_async.severity = warning dotnet_naming_rule.async_methods_should_be_ends_with_async.symbols = async_methods dotnet_naming_rule.async_methods_should_be_ends_with_async.style = ends_with_async - dotnet_naming_rule.methods__events__delegates_and_local_functions_should_be_pascalcase.severity = error dotnet_naming_rule.methods__events__delegates_and_local_functions_should_be_pascalcase.symbols = methods__events__delegates_and_local_functions dotnet_naming_rule.methods__events__delegates_and_local_functions_should_be_pascalcase.style = pascalcase - dotnet_naming_rule.properties_should_be_pascalcase.severity = error dotnet_naming_rule.properties_should_be_pascalcase.symbols = properties dotnet_naming_rule.properties_should_be_pascalcase.style = pascalcase - dotnet_naming_rule.generics_should_be_begins_with_t.severity = warning dotnet_naming_rule.generics_should_be_begins_with_t.symbols = generics dotnet_naming_rule.generics_should_be_begins_with_t.style = begins_with_t - dotnet_naming_rule.constants_should_be_pascalcase.severity = error dotnet_naming_rule.constants_should_be_pascalcase.symbols = constants dotnet_naming_rule.constants_should_be_pascalcase.style = pascalcase - dotnet_naming_rule.public_fields_should_be_pascalcase.severity = error dotnet_naming_rule.public_fields_should_be_pascalcase.symbols = public_fields dotnet_naming_rule.public_fields_should_be_pascalcase.style = pascalcase - dotnet_naming_rule.fields__parameters_and_locals_should_be_camelcase.severity = error dotnet_naming_rule.fields__parameters_and_locals_should_be_camelcase.symbols = fields__parameters_and_locals dotnet_naming_rule.fields__parameters_and_locals_should_be_camelcase.style = camelcase - dotnet_naming_rule.namespaces_should_be_pascalcase.severity = warning dotnet_naming_rule.namespaces_should_be_pascalcase.symbols = namespaces dotnet_naming_rule.namespaces_should_be_pascalcase.style = pascalcase - -# Symbol specifications - dotnet_naming_symbols.namespaces.applicable_kinds = namespace dotnet_naming_symbols.namespaces.applicable_accessibilities = * -dotnet_naming_symbols.namespaces.required_modifiers = - dotnet_naming_symbols.classes.applicable_kinds = class dotnet_naming_symbols.classes.applicable_accessibilities = * -dotnet_naming_symbols.classes.required_modifiers = - dotnet_naming_symbols.structures_and_enums.applicable_kinds = struct, enum dotnet_naming_symbols.structures_and_enums.applicable_accessibilities = * -dotnet_naming_symbols.structures_and_enums.required_modifiers = - dotnet_naming_symbols.interfaces.applicable_kinds = interface dotnet_naming_symbols.interfaces.applicable_accessibilities = * -dotnet_naming_symbols.interfaces.required_modifiers = - dotnet_naming_symbols.properties.applicable_kinds = property dotnet_naming_symbols.properties.applicable_accessibilities = * -dotnet_naming_symbols.properties.required_modifiers = - dotnet_naming_symbols.methods__events__delegates_and_local_functions.applicable_kinds = event, delegate, method, local_function dotnet_naming_symbols.methods__events__delegates_and_local_functions.applicable_accessibilities = * -dotnet_naming_symbols.methods__events__delegates_and_local_functions.required_modifiers = - dotnet_naming_symbols.fields__parameters_and_locals.applicable_kinds = field, parameter, local dotnet_naming_symbols.fields__parameters_and_locals.applicable_accessibilities = internal, private, protected, protected_internal, private_protected, local -dotnet_naming_symbols.fields__parameters_and_locals.required_modifiers = - dotnet_naming_symbols.generics.applicable_kinds = type_parameter dotnet_naming_symbols.generics.applicable_accessibilities = * -dotnet_naming_symbols.generics.required_modifiers = - dotnet_naming_symbols.constants.applicable_kinds = field dotnet_naming_symbols.constants.applicable_accessibilities = * dotnet_naming_symbols.constants.required_modifiers = const - dotnet_naming_symbols.async_methods.applicable_kinds = method dotnet_naming_symbols.async_methods.applicable_accessibilities = * dotnet_naming_symbols.async_methods.required_modifiers = async - dotnet_naming_symbols.public_fields.applicable_kinds = field dotnet_naming_symbols.public_fields.applicable_accessibilities = public -dotnet_naming_symbols.public_fields.required_modifiers = - -# Naming styles - -dotnet_naming_style.pascalcase.required_prefix = -dotnet_naming_style.pascalcase.required_suffix = -dotnet_naming_style.pascalcase.word_separator = dotnet_naming_style.pascalcase.capitalization = pascal_case - dotnet_naming_style.begins_with_i.required_prefix = I -dotnet_naming_style.begins_with_i.required_suffix = -dotnet_naming_style.begins_with_i.word_separator = dotnet_naming_style.begins_with_i.capitalization = pascal_case - -dotnet_naming_style.camelcase.required_prefix = -dotnet_naming_style.camelcase.required_suffix = -dotnet_naming_style.camelcase.word_separator = dotnet_naming_style.camelcase.capitalization = camel_case - dotnet_naming_style.begins_with_t.required_prefix = T -dotnet_naming_style.begins_with_t.required_suffix = -dotnet_naming_style.begins_with_t.word_separator = dotnet_naming_style.begins_with_t.capitalization = pascal_case - -dotnet_naming_style.ends_with_async.required_prefix = dotnet_naming_style.ends_with_async.required_suffix = Async -dotnet_naming_style.ends_with_async.word_separator = dotnet_naming_style.ends_with_async.capitalization = pascal_case + diff --git a/Directory.Build.props b/Directory.Build.props index 15961fb..d9ac681 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,63 +4,17 @@ Note for future self: - Any property defined here can be explicitly overridden in any of said projects. --> - - - 3 - 3.0.0 - prerelease1 - TODO - + + - - - enable - 14 - AnyCPU;ARM64;x64;x86;ARM32 - true + + + - - OceanApocalypse - OceanApocalypse - Copyright 2025-2026 OceanApocalypse - Apache-2.0 - git - true - en - + - - - latest-all - All - - - - - - $(NoWarn);CA1016;CA1034;CA1043;CA1708 - - - - - - - - - - - 0 - - - - - $(SemVerMajor).0.0.0 - $(VersionPrefix).$(GITHUB_RUN_NUMBER) - + \ No newline at end of file diff --git a/RedSeaModernLanguage.slnx b/RedSeaModernLanguage.slnx index aafaf17..ac86c85 100644 --- a/RedSeaModernLanguage.slnx +++ b/RedSeaModernLanguage.slnx @@ -31,31 +31,43 @@ - + - + - + - + + + + + + + - + + + + + + + diff --git a/benchmarks/RSML.Benchmarks.csproj b/benchmarks/RSML.Benchmarks.csproj index 75c60e2..55ceccb 100644 --- a/benchmarks/RSML.Benchmarks.csproj +++ b/benchmarks/RSML.Benchmarks.csproj @@ -1,8 +1,6 @@  - net10.0;net8.0 Exe - enable OceanApocalypse.RSML.Benchmarks pdbonly true @@ -18,7 +16,7 @@ - + \ No newline at end of file diff --git a/props/Analyzers.props b/props/Analyzers.props new file mode 100644 index 0000000..4abd312 --- /dev/null +++ b/props/Analyzers.props @@ -0,0 +1,15 @@ + + + + latest-all + All + + + + + + $(NoWarn);CA1016;CA1034;CA1043;CA1708 + + \ No newline at end of file diff --git a/props/Common.props b/props/Common.props new file mode 100644 index 0000000..5fa3409 --- /dev/null +++ b/props/Common.props @@ -0,0 +1,36 @@ + + + + enable + 14 + AnyCPU;ARM64;x64;x86;ARM32 + true + + $(PackageId) + + + + full + + + + none + true + + + + + + + + + + 0 + + \ No newline at end of file diff --git a/props/Defaults.props b/props/Defaults.props new file mode 100644 index 0000000..06a0955 --- /dev/null +++ b/props/Defaults.props @@ -0,0 +1,9 @@ + + + + net10.0;net8.0 + true + + true + + \ No newline at end of file diff --git a/props/NuGet.props b/props/NuGet.props new file mode 100644 index 0000000..9308b7a --- /dev/null +++ b/props/NuGet.props @@ -0,0 +1,23 @@ + + + OceanApocalypse + OceanApocalypse + Copyright 2025-2026 OceanApocalypse + Apache-2.0 + + The only DSL that dynamically interprets different logic paths based on an + host's OS and CPU architecture. + + https://github.com/OceanApocalypse/RedSeaModernLanguage/ + https://oceanapocalypse.org/rsml-docs/ + + DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss + README.md + icon.png + + git + true + en + + + \ No newline at end of file diff --git a/props/README.md b/props/README.md new file mode 100644 index 0000000..8b5ebb2 --- /dev/null +++ b/props/README.md @@ -0,0 +1,47 @@ +# Properties set by the `.props` files in this directory +- `Common.props` + - **Nullable context:** enabled + - **Language version:** 14 + - **Platforms:** any CPU architecture, `arm64`, `arm`, `x64` and `x86` + - **Overflow/underflow checks:** enabled + - **Root namespace:** same as package ID, unless package ID is unset + - **Debug configuration** + - **Debug type:** full + - **Release configuration** + - **Debug type:** none + - **Optimizations:** enabled + - **Resources** + - **[/assets/icon.png](/assets/icon.png):** packed into package root; not visible in Solution Explorer + - **[/README.md](/README.md):** packed into package root; not visible in Solution Explorer + - **`GITHUB_RUN_NUMBER`:** 0 unless already set in the environment - meant for local build support (see `Versioning.props`) +- `Default.props` + - **Target frameworks:** .NET 10.0 and .NET 8.0 + - **AOT compatibility:** enabled + - **Generate documentation file on build:** yes +- `Analyzers.props` + - **Analysis level:** set to `latest-all` + - **Analysis mode:** set to `All` + - **Disabled warnings:** CA1016, CA1034, CA1043 and CA1708, alongside the warnings disabled by default +- `Versioning.props` + - **`SemVerMajor`** _(must be changed every new **major** version)_ + - **`VersionPrefix`** _(must be changed every new version)_ + - **`VersionSuffix`** _(must be changed every new version)_ + - **`PackageReleaseNotes`** _(must be changed every new version)_ + - **Assembly metadata:** a `SemVersion` injected into the assembly metadata, taking the `VersionPrefix` form if `VersionSuffix` is unset; otherwise, takes the form `VersionPrefix-VersionSuffix`, mimicking how `Version` is set internally by MSBuild + - **Assembly version:** set to `SemVerMajor.0.0.0` **always** + - **File version:** set to `VersionPrefix.GITHUB_RUN_NUMBER` **always** +- `NuGet.props` + - **Authors:** OceanApocalypse + - **Company:** OceanApocalypse + - **Copyright message:** Copyright 2025-2026 OceanApocalypse + - **Package license:** Apache 2.0 + - **Description:** set to `The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture.` + - **Repo URL:** https://github.com/OceanApocalypse/RedSeaModernLanguage/ + - **Package project URL:** https://oceanapocalypse.org/rsml-docs/ + - **Package tags:** set to `DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss` + - **Package README file:** set to previously imported `README.md` (see `Common.props`) + - **Package icon file:** set to previously imported `icon.png` (see `Common.props`) + - **Repo type:** git + - **Package requires license acceptance:** yes + - **Neutral language:** English (`en`) + diff --git a/props/Versioning.props b/props/Versioning.props new file mode 100644 index 0000000..6616b50 --- /dev/null +++ b/props/Versioning.props @@ -0,0 +1,25 @@ + + + + + 3 + 3.0.0 + prerelease1 + TODO + + + + + + + + + + + $(SemVerMajor).0.0.0 + $(VersionPrefix).$(GITHUB_RUN_NUMBER) + + \ No newline at end of file diff --git a/src/RSML.Abstractions.Diagnostics/RSML.Abstractions.Diagnostics.csproj b/src/RSML.Abstractions.Diagnostics/RSML.Abstractions.Diagnostics.csproj new file mode 100644 index 0000000..78bb2b7 --- /dev/null +++ b/src/RSML.Abstractions.Diagnostics/RSML.Abstractions.Diagnostics.csproj @@ -0,0 +1,18 @@ + + + + true + Library + OceanApocalypse.RSML.Abstractions.Diagnostics + + + + True + OceanApocalypse.RSML.Abstractions.Diagnostics + RSML Diagnostics + + + + + + \ No newline at end of file diff --git a/src/RSML.Abstractions.Panic/RSML.Abstractions.Panic.csproj b/src/RSML.Abstractions.Panic/RSML.Abstractions.Panic.csproj new file mode 100644 index 0000000..15d36a4 --- /dev/null +++ b/src/RSML.Abstractions.Panic/RSML.Abstractions.Panic.csproj @@ -0,0 +1,14 @@ + + + + true + Library + OceanApocalypse.RSML.Abstractions.Panic + + + + True + OceanApocalypse.RSML.Abstractions.Panic + RSML Panic + + \ No newline at end of file diff --git a/src/RSML.Abstractions/RSML.Abstractions.csproj b/src/RSML.Abstractions/RSML.Abstractions.csproj new file mode 100644 index 0000000..fa222eb --- /dev/null +++ b/src/RSML.Abstractions/RSML.Abstractions.csproj @@ -0,0 +1,18 @@ + + + true + Library + OceanApocalypse.RSML.Abstractions + + + + True + OceanApocalypse.RSML.Abstractions + RSML Abstractions + + + + + + + \ No newline at end of file diff --git a/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj b/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj index f9a6912..18ab05f 100644 --- a/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj +++ b/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj @@ -1,54 +1,20 @@ - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Language.Lexing True - OceanApocalypse.RSML.Language.Lexing RSML Lexers - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - - - + + + diff --git a/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj b/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj index 16ada04..a303b29 100644 --- a/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj +++ b/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj @@ -1,54 +1,18 @@ - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Language.Parsing True - OceanApocalypse.RSML.Language.Parsing RSML Parsers - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - - - + diff --git a/src/RSML.Native/RSML.Native.csproj b/src/RSML.Native/RSML.Native.csproj index 75d6978..922e5ba 100644 --- a/src/RSML.Native/RSML.Native.csproj +++ b/src/RSML.Native/RSML.Native.csproj @@ -2,7 +2,6 @@ net10.0 - true OceanApocalypse.RSML.Native Library @@ -20,26 +19,9 @@ RSML Native ABI Native C interopability for RSML. - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ DSL;native;interop;c;cpp;c++;language;logic;logic-path;rsml;rsea;decision;oss - README.md - - - True - \ - false - - - True - \ - false - - - diff --git a/src/RSML.Toolchain.Abstractions/RSML.Toolchain.Abstractions.csproj b/src/RSML.Toolchain.Abstractions/RSML.Toolchain.Abstractions.csproj deleted file mode 100644 index f9933a7..0000000 --- a/src/RSML.Toolchain.Abstractions/RSML.Toolchain.Abstractions.csproj +++ /dev/null @@ -1,50 +0,0 @@ - - - - net10.0;net8.0 - true - true - - Library - true - - OceanApocalypse.RSML.Toolchain.Abstractions - - - - True - - OceanApocalypse.RSML.Toolchain.Abstractions - RSML Abstractions - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - - diff --git a/src/RSML.Toolchain.Analysis/RSML.Toolchain.Analysis.csproj b/src/RSML.Toolchain.Analysis/RSML.Toolchain.Analysis.csproj index 8c83186..c2d7293 100644 --- a/src/RSML.Toolchain.Analysis/RSML.Toolchain.Analysis.csproj +++ b/src/RSML.Toolchain.Analysis/RSML.Toolchain.Analysis.csproj @@ -1,50 +1,14 @@  - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Toolchain.Analysis True - OceanApocalypse.RSML.Toolchain.Analysis RSML Analyzers - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - diff --git a/src/RSML.Toolchain.Execution/RSML.Toolchain.Execution.csproj b/src/RSML.Toolchain.Execution/RSML.Toolchain.Execution.csproj index 1b067ae..3c7f23f 100644 --- a/src/RSML.Toolchain.Execution/RSML.Toolchain.Execution.csproj +++ b/src/RSML.Toolchain.Execution/RSML.Toolchain.Execution.csproj @@ -1,50 +1,14 @@ - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Toolchain.Execution True - OceanApocalypse.RSML.Toolchain.Execution RSML Interpreters - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - diff --git a/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs b/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs index 9b978db..f0f1685 100644 --- a/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs +++ b/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs @@ -1,6 +1,6 @@ using System; -using OceanApocalypse.RSML.Toolchain.Abstractions; +using OceanApocalypse.RSML.Abstractions; namespace OceanApocalypse.RSML.Toolchain.Extensibility.Execution; diff --git a/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj b/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj index 6cef39b..6167081 100644 --- a/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj +++ b/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj @@ -1,54 +1,17 @@ - - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Toolchain.Extensibility.Execution True - OceanApocalypse.RSML.Toolchain.Extensibility.Execution RSML Extensible Interpreters - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - - + - + diff --git a/src/RSML.Toolchain.Extensibility.Lexing/RSML.Toolchain.Extensibility.Lexing.csproj b/src/RSML.Toolchain.Extensibility.Lexing/RSML.Toolchain.Extensibility.Lexing.csproj index 707d5b7..c656e97 100644 --- a/src/RSML.Toolchain.Extensibility.Lexing/RSML.Toolchain.Extensibility.Lexing.csproj +++ b/src/RSML.Toolchain.Extensibility.Lexing/RSML.Toolchain.Extensibility.Lexing.csproj @@ -1,50 +1,14 @@ - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Toolchain.Extensibility.Lexing True - OceanApocalypse.RSML.Toolchain.Extensibility.Lexing RSML Extensible Lexers - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - diff --git a/src/RSML.Toolchain.Extensibility.Parsing/RSML.Toolchain.Extensibility.Parsing.csproj b/src/RSML.Toolchain.Extensibility.Parsing/RSML.Toolchain.Extensibility.Parsing.csproj index ca28eb9..6261a08 100644 --- a/src/RSML.Toolchain.Extensibility.Parsing/RSML.Toolchain.Extensibility.Parsing.csproj +++ b/src/RSML.Toolchain.Extensibility.Parsing/RSML.Toolchain.Extensibility.Parsing.csproj @@ -1,50 +1,13 @@ - - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Toolchain.Extensibility.Parsing True - OceanApocalypse.RSML.Toolchain.Extensibility.Parsing RSML Extensible Parsers - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - diff --git a/src/RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj b/src/RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj index 93ce1f8..09854eb 100644 --- a/src/RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj +++ b/src/RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj @@ -1,54 +1,19 @@ - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML.Toolchain.Sources True - OceanApocalypse.RSML.Toolchain.Sources RSML Sources - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - full - - - - none - true - - - - - True - \ - false - - - True - \ - false - - - - + + diff --git a/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs b/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs index f85e2aa..50afe4c 100644 --- a/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs +++ b/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs @@ -4,10 +4,10 @@ using System.Runtime.CompilerServices; using System.Text; -using OceanApocalypse.RSML.Toolchain.Abstractions; -using OceanApocalypse.RSML.Toolchain.Abstractions.Cache; -using OceanApocalypse.RSML.Toolchain.Abstractions.Panic; -using OceanApocalypse.RSML.Toolchain.Abstractions.Sources; +using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Cache; +using OceanApocalypse.RSML.Abstractions.Panic; +using OceanApocalypse.RSML.Abstractions.Sources; namespace OceanApocalypse.RSML.Toolchain.Sources; diff --git a/src/RSML/RSML.csproj b/src/RSML/RSML.csproj index c2ad29d..a55037c 100644 --- a/src/RSML/RSML.csproj +++ b/src/RSML/RSML.csproj @@ -1,13 +1,7 @@ - - net10.0;net8.0 true - true - Library - true - OceanApocalypse.RSML @@ -16,40 +10,5 @@ OceanApocalypse.RSML Red Sea Modern Language (RSML) - The only DSL that dynamically interprets different logic paths based on an host's OS and CPU architecture. - - https://OceanApocalypse.org/rsml-docs/ - icon.png - https://github.com/OceanApocalypse/RedSeaModernLanguage/ - - DSL;language;logic;logic-path;system;host;rsml;rsea;decision;oss - README.md - - - - full - True - - - - none - true - - - - True - \ - false - - - True - \ - false - - - - - - diff --git a/tests/RSML.InternalTests/RSML.InternalTests.csproj b/tests/RSML.InternalTests/RSML.InternalTests.csproj index 61e87b0..728ce7f 100644 --- a/tests/RSML.InternalTests/RSML.InternalTests.csproj +++ b/tests/RSML.InternalTests/RSML.InternalTests.csproj @@ -2,13 +2,10 @@ net10.0 - enable Exe true OceanApocalypse.RSML.Tests.Internal - latest true - AnyCPU;ARM64;x64;x86;ARM32 diff --git a/tests/RSML.NativeTests/RSML.NativeTests.csproj b/tests/RSML.NativeTests/RSML.NativeTests.csproj index 468dc09..32e951a 100644 --- a/tests/RSML.NativeTests/RSML.NativeTests.csproj +++ b/tests/RSML.NativeTests/RSML.NativeTests.csproj @@ -2,14 +2,10 @@ net10.0 - enable Exe - enable - OceanApocalypse.RSML.Tests.Native true - latest + OceanApocalypse.RSML.Tests.Native true - AnyCPU;ARM64;x64;x86;ARM32 diff --git a/tests/RSML.Tests/RSML.Tests.csproj b/tests/RSML.Tests/RSML.Tests.csproj index 725e13b..8c2d984 100644 --- a/tests/RSML.Tests/RSML.Tests.csproj +++ b/tests/RSML.Tests/RSML.Tests.csproj @@ -2,13 +2,10 @@ net10.0 - enable Exe true OceanApocalypse.RSML.Tests - latest true - AnyCPU;ARM64;x64;x86;ARM32 @@ -25,8 +22,9 @@ - + + diff --git a/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs b/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs index 2080022..761a117 100644 --- a/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs +++ b/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs @@ -1,8 +1,7 @@ using System; using System.Text; -using OceanApocalypse.RSML.Toolchain.Abstractions.Panic; -using OceanApocalypse.RSML.Toolchain.Abstractions.Sources; +using OceanApocalypse.RSML.Abstractions.Panic; using OceanApocalypse.RSML.Toolchain.Sources; namespace OceanApocalypse.RSML.Tests.Sources; From b66f26b98f065e9982c98933079f5b6447d78af1 Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 9 Aug 2026 23:42:17 +0100 Subject: [PATCH 05/11] refactor(abstractions)!: Move abstractions to RSML.Abstractions project and split it by diagnostics and panics (soon by sources as well) --- .../AssemblyInfo.cs | 0 .../Diagnostic.cs | 4 +-- .../DiagnosticCollector.cs | 10 +++++-- .../ErrorCodes/InternalErrorCodes.cs | 2 +- .../GlobalSuppressions.cs | 8 +++++ .../Result.cs | 4 +-- .../Result{TValue}.cs | 2 +- .../Severity.cs | 2 +- .../BufferException.cs | 4 +-- .../ExceededMaxAmountOfFailures.cs | 30 +++++++++++++++++++ src/RSML.Abstractions.Panic/LexerException.cs | 28 +++++++++++++++++ src/RSML.Abstractions/AssemblyInfo.cs | 16 ++++++++++ .../Cache/ISupportsCache.cs | 2 +- .../Extensions.cs | 2 +- src/RSML.Abstractions/GlobalSuppressions.cs | 7 +++++ .../IToolchainComponent.cs | 2 +- .../Sources/IBuffer.cs | 2 +- .../Sources/IScanner.cs | 2 +- .../ToolchainConfigurations.cs | 2 +- src/RSML.Language.Parsing/IParser.cs | 2 +- src/RSML.Language.Parsing/Parser.cs | 2 +- .../GlobalSuppressions.cs | 8 ----- 22 files changed, 112 insertions(+), 29 deletions(-) rename src/{RSML.Toolchain.Abstractions => RSML.Abstractions.Diagnostics}/AssemblyInfo.cs (100%) rename src/{RSML.Toolchain.Abstractions/Diagnostics => RSML.Abstractions.Diagnostics}/Diagnostic.cs (98%) rename src/{RSML.Toolchain.Abstractions/Diagnostics => RSML.Abstractions.Diagnostics}/DiagnosticCollector.cs (84%) rename src/{RSML.Toolchain.Abstractions/Diagnostics => RSML.Abstractions.Diagnostics}/ErrorCodes/InternalErrorCodes.cs (71%) create mode 100644 src/RSML.Abstractions.Diagnostics/GlobalSuppressions.cs rename src/{RSML.Toolchain.Abstractions/Diagnostics => RSML.Abstractions.Diagnostics}/Result.cs (96%) rename src/{RSML.Toolchain.Abstractions/Diagnostics => RSML.Abstractions.Diagnostics}/Result{TValue}.cs (98%) rename src/{RSML.Toolchain.Abstractions/Diagnostics => RSML.Abstractions.Diagnostics}/Severity.cs (88%) rename src/{RSML.Toolchain.Abstractions/Panic => RSML.Abstractions.Panic}/BufferException.cs (83%) create mode 100644 src/RSML.Abstractions.Panic/ExceededMaxAmountOfFailures.cs create mode 100644 src/RSML.Abstractions.Panic/LexerException.cs create mode 100644 src/RSML.Abstractions/AssemblyInfo.cs rename src/{RSML.Toolchain.Abstractions => RSML.Abstractions}/Cache/ISupportsCache.cs (91%) rename src/{RSML.Toolchain.Abstractions => RSML.Abstractions}/Extensions.cs (96%) create mode 100644 src/RSML.Abstractions/GlobalSuppressions.cs rename src/{RSML.Toolchain.Abstractions => RSML.Abstractions}/IToolchainComponent.cs (90%) rename src/{RSML.Toolchain.Abstractions => RSML.Abstractions}/Sources/IBuffer.cs (99%) rename src/{RSML.Toolchain.Abstractions => RSML.Abstractions}/Sources/IScanner.cs (88%) rename src/{RSML.Toolchain.Abstractions => RSML.Abstractions}/ToolchainConfigurations.cs (96%) delete mode 100644 src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs diff --git a/src/RSML.Toolchain.Abstractions/AssemblyInfo.cs b/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs similarity index 100% rename from src/RSML.Toolchain.Abstractions/AssemblyInfo.cs rename to src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs diff --git a/src/RSML.Toolchain.Abstractions/Diagnostics/Diagnostic.cs b/src/RSML.Abstractions.Diagnostics/Diagnostic.cs similarity index 98% rename from src/RSML.Toolchain.Abstractions/Diagnostics/Diagnostic.cs rename to src/RSML.Abstractions.Diagnostics/Diagnostic.cs index a13c4ed..cffb462 100644 --- a/src/RSML.Toolchain.Abstractions/Diagnostics/Diagnostic.cs +++ b/src/RSML.Abstractions.Diagnostics/Diagnostic.cs @@ -1,10 +1,8 @@ using System; using System.Diagnostics.CodeAnalysis; -using OceanApocalypse.RSML.Toolchain.Abstractions.Sources; - -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; +namespace OceanApocalypse.RSML.Abstractions.Diagnostics; /// /// A diagnostic reported by RSML's API. diff --git a/src/RSML.Toolchain.Abstractions/Diagnostics/DiagnosticCollector.cs b/src/RSML.Abstractions.Diagnostics/DiagnosticCollector.cs similarity index 84% rename from src/RSML.Toolchain.Abstractions/Diagnostics/DiagnosticCollector.cs rename to src/RSML.Abstractions.Diagnostics/DiagnosticCollector.cs index 0737d78..5235d50 100644 --- a/src/RSML.Toolchain.Abstractions/Diagnostics/DiagnosticCollector.cs +++ b/src/RSML.Abstractions.Diagnostics/DiagnosticCollector.cs @@ -2,19 +2,23 @@ using System.Collections.Generic; using System.Collections.Immutable; - -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; +namespace OceanApocalypse.RSML.Abstractions.Diagnostics; /// /// A list of RSML toolchain errors. /// public sealed record DiagnosticCollector : IEnumerable { + /// + /// Creates a new diagnostic collector. + /// + public DiagnosticCollector() => MinimumCriticalSeverity = Severity.Error; + /// /// Creates a new diagnostic collector. /// /// The minimum diagnostic severity for a diagnostic to be considered critical. - public DiagnosticCollector(Severity minimumCriticalSeverity = Severity.Error) => MinimumCriticalSeverity = minimumCriticalSeverity; + public DiagnosticCollector(Severity minimumCriticalSeverity) => MinimumCriticalSeverity = minimumCriticalSeverity; private readonly List diagnostics = []; diff --git a/src/RSML.Toolchain.Abstractions/Diagnostics/ErrorCodes/InternalErrorCodes.cs b/src/RSML.Abstractions.Diagnostics/ErrorCodes/InternalErrorCodes.cs similarity index 71% rename from src/RSML.Toolchain.Abstractions/Diagnostics/ErrorCodes/InternalErrorCodes.cs rename to src/RSML.Abstractions.Diagnostics/ErrorCodes/InternalErrorCodes.cs index 42e1b3c..9e8f486 100644 --- a/src/RSML.Toolchain.Abstractions/Diagnostics/ErrorCodes/InternalErrorCodes.cs +++ b/src/RSML.Abstractions.Diagnostics/ErrorCodes/InternalErrorCodes.cs @@ -1,4 +1,4 @@ -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics.ErrorCodes; +namespace OceanApocalypse.RSML.Abstractions.Diagnostics.ErrorCodes; internal static class InternalErrorCodes { diff --git a/src/RSML.Abstractions.Diagnostics/GlobalSuppressions.cs b/src/RSML.Abstractions.Diagnostics/GlobalSuppressions.cs new file mode 100644 index 0000000..d7490f6 --- /dev/null +++ b/src/RSML.Abstractions.Diagnostics/GlobalSuppressions.cs @@ -0,0 +1,8 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("Style", "IDE0305:Simplify collection initialization", Justification = "", Scope = "member", Target = "~M:OceanApocalypse.RSML.Abstractions.Diagnostics.DiagnosticCollector.GetAll~System.Collections.Immutable.ImmutableArray{OceanApocalypse.RSML.Abstractions.Diagnostics.Diagnostic}")] diff --git a/src/RSML.Toolchain.Abstractions/Diagnostics/Result.cs b/src/RSML.Abstractions.Diagnostics/Result.cs similarity index 96% rename from src/RSML.Toolchain.Abstractions/Diagnostics/Result.cs rename to src/RSML.Abstractions.Diagnostics/Result.cs index 1d8f707..68700f3 100644 --- a/src/RSML.Toolchain.Abstractions/Diagnostics/Result.cs +++ b/src/RSML.Abstractions.Diagnostics/Result.cs @@ -1,8 +1,8 @@ using System; -using OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics.ErrorCodes; +using OceanApocalypse.RSML.Abstractions.Diagnostics.ErrorCodes; -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; +namespace OceanApocalypse.RSML.Abstractions.Diagnostics; /// /// A collection of factory methods for easier initialization of objects. diff --git a/src/RSML.Toolchain.Abstractions/Diagnostics/Result{TValue}.cs b/src/RSML.Abstractions.Diagnostics/Result{TValue}.cs similarity index 98% rename from src/RSML.Toolchain.Abstractions/Diagnostics/Result{TValue}.cs rename to src/RSML.Abstractions.Diagnostics/Result{TValue}.cs index 82215b3..c47d0e7 100644 --- a/src/RSML.Toolchain.Abstractions/Diagnostics/Result{TValue}.cs +++ b/src/RSML.Abstractions.Diagnostics/Result{TValue}.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; +namespace OceanApocalypse.RSML.Abstractions.Diagnostics; /// /// An operation's result. diff --git a/src/RSML.Toolchain.Abstractions/Diagnostics/Severity.cs b/src/RSML.Abstractions.Diagnostics/Severity.cs similarity index 88% rename from src/RSML.Toolchain.Abstractions/Diagnostics/Severity.cs rename to src/RSML.Abstractions.Diagnostics/Severity.cs index f79445b..f813325 100644 --- a/src/RSML.Toolchain.Abstractions/Diagnostics/Severity.cs +++ b/src/RSML.Abstractions.Diagnostics/Severity.cs @@ -1,4 +1,4 @@ -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; +namespace OceanApocalypse.RSML.Abstractions.Diagnostics; /// /// The severity of an error. diff --git a/src/RSML.Toolchain.Abstractions/Panic/BufferException.cs b/src/RSML.Abstractions.Panic/BufferException.cs similarity index 83% rename from src/RSML.Toolchain.Abstractions/Panic/BufferException.cs rename to src/RSML.Abstractions.Panic/BufferException.cs index 6313fe5..0324bae 100644 --- a/src/RSML.Toolchain.Abstractions/Panic/BufferException.cs +++ b/src/RSML.Abstractions.Panic/BufferException.cs @@ -1,10 +1,10 @@ using System; using System.IO; -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Panic; +namespace OceanApocalypse.RSML.Abstractions.Panic; /// -/// An exception that occurs in and types. +/// An exception that occurs in buffer types. /// public class BufferException : IOException { diff --git a/src/RSML.Abstractions.Panic/ExceededMaxAmountOfFailures.cs b/src/RSML.Abstractions.Panic/ExceededMaxAmountOfFailures.cs new file mode 100644 index 0000000..3e47875 --- /dev/null +++ b/src/RSML.Abstractions.Panic/ExceededMaxAmountOfFailures.cs @@ -0,0 +1,30 @@ +using System; + +namespace OceanApocalypse.RSML.Abstractions.Panic; + +/// +/// An exception that is thrown when a given operation exceeds the maximum +/// amount of internal failures it is allowed to endure. +/// This class cannot be inherited. +/// +public sealed class ExceededMaxAmountOfFailuresException : Exception +{ + /// + /// Creates a new exception of this type with no message. + /// + public ExceededMaxAmountOfFailuresException() : base() { } + + /// + /// Creates a new exception of this type with a custom error message. + /// + /// The error message. + public ExceededMaxAmountOfFailuresException(string message) : base(message) { } + + /// + /// Creates a new exception of this type with a custom error message and + /// a reference to the exception that caused this panic. + /// + /// The error message. + /// The exception that led to the panic. + public ExceededMaxAmountOfFailuresException(string? message, Exception innerException) : base(message, innerException) { } +} diff --git a/src/RSML.Abstractions.Panic/LexerException.cs b/src/RSML.Abstractions.Panic/LexerException.cs new file mode 100644 index 0000000..3e00575 --- /dev/null +++ b/src/RSML.Abstractions.Panic/LexerException.cs @@ -0,0 +1,28 @@ +using System; + +namespace OceanApocalypse.RSML.Abstractions.Panic; + +/// +/// An exception that occurs in lexer types. +/// +public class LexerException : Exception +{ + /// + /// Creates a new lexer exception with no message. + /// + public LexerException() : base() { } + + /// + /// Creates a new lexer exception with a custom error message. + /// + /// The error message. + public LexerException(string message) : base(message) { } + + /// + /// Creates a new lexer exception with a custom error message and a reference + /// to the exception that caused this panic. + /// + /// The error message. + /// The exception that led to the panic. + public LexerException(string? message, Exception innerException) : base(message, innerException) { } +} diff --git a/src/RSML.Abstractions/AssemblyInfo.cs b/src/RSML.Abstractions/AssemblyInfo.cs new file mode 100644 index 0000000..3fa1539 --- /dev/null +++ b/src/RSML.Abstractions/AssemblyInfo.cs @@ -0,0 +1,16 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// In SDK-style projects such as this one, several assembly attributes that were historically +// defined in this file are now automatically added during build and populated with +// values defined in project properties. For details of which attributes are included +// and how to customise this process see: https://aka.ms/assembly-info-properties + +// Setting ComVisible to false makes the types in this assembly not visible to COM +// components. If you need to access a type in this assembly from COM, set the ComVisible +// attribute to true on that type. +[assembly: ComVisible(false)] + +[assembly: CLSCompliant(true)] +[assembly: InternalsVisibleTo("RSML.Toolchain.Sources")] diff --git a/src/RSML.Toolchain.Abstractions/Cache/ISupportsCache.cs b/src/RSML.Abstractions/Cache/ISupportsCache.cs similarity index 91% rename from src/RSML.Toolchain.Abstractions/Cache/ISupportsCache.cs rename to src/RSML.Abstractions/Cache/ISupportsCache.cs index f764851..6269556 100644 --- a/src/RSML.Toolchain.Abstractions/Cache/ISupportsCache.cs +++ b/src/RSML.Abstractions/Cache/ISupportsCache.cs @@ -1,4 +1,4 @@ -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Cache; +namespace OceanApocalypse.RSML.Abstractions.Cache; /// /// Represents a service or a type that supports cached data. diff --git a/src/RSML.Toolchain.Abstractions/Extensions.cs b/src/RSML.Abstractions/Extensions.cs similarity index 96% rename from src/RSML.Toolchain.Abstractions/Extensions.cs rename to src/RSML.Abstractions/Extensions.cs index 9ce4725..e85290c 100644 --- a/src/RSML.Toolchain.Abstractions/Extensions.cs +++ b/src/RSML.Abstractions/Extensions.cs @@ -2,7 +2,7 @@ using System.Collections.Immutable; using System.Runtime.CompilerServices; -namespace OceanApocalypse.RSML.Toolchain.Abstractions; +namespace OceanApocalypse.RSML.Abstractions; /// /// Extension members for characters. diff --git a/src/RSML.Abstractions/GlobalSuppressions.cs b/src/RSML.Abstractions/GlobalSuppressions.cs new file mode 100644 index 0000000..a1fe9a6 --- /dev/null +++ b/src/RSML.Abstractions/GlobalSuppressions.cs @@ -0,0 +1,7 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + diff --git a/src/RSML.Toolchain.Abstractions/IToolchainComponent.cs b/src/RSML.Abstractions/IToolchainComponent.cs similarity index 90% rename from src/RSML.Toolchain.Abstractions/IToolchainComponent.cs rename to src/RSML.Abstractions/IToolchainComponent.cs index 0969882..d9f1dad 100644 --- a/src/RSML.Toolchain.Abstractions/IToolchainComponent.cs +++ b/src/RSML.Abstractions/IToolchainComponent.cs @@ -1,6 +1,6 @@ using System; -namespace OceanApocalypse.RSML.Toolchain.Abstractions; +namespace OceanApocalypse.RSML.Abstractions; /// /// A component of the RSML toolchain. diff --git a/src/RSML.Toolchain.Abstractions/Sources/IBuffer.cs b/src/RSML.Abstractions/Sources/IBuffer.cs similarity index 99% rename from src/RSML.Toolchain.Abstractions/Sources/IBuffer.cs rename to src/RSML.Abstractions/Sources/IBuffer.cs index d80d1f7..b72c845 100644 --- a/src/RSML.Toolchain.Abstractions/Sources/IBuffer.cs +++ b/src/RSML.Abstractions/Sources/IBuffer.cs @@ -1,6 +1,6 @@ using System; -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Sources; +namespace OceanApocalypse.RSML.Abstractions.Sources; /// /// Represents a buffer of characters. diff --git a/src/RSML.Toolchain.Abstractions/Sources/IScanner.cs b/src/RSML.Abstractions/Sources/IScanner.cs similarity index 88% rename from src/RSML.Toolchain.Abstractions/Sources/IScanner.cs rename to src/RSML.Abstractions/Sources/IScanner.cs index 043762c..a71b742 100644 --- a/src/RSML.Toolchain.Abstractions/Sources/IScanner.cs +++ b/src/RSML.Abstractions/Sources/IScanner.cs @@ -1,7 +1,7 @@ using System; -namespace OceanApocalypse.RSML.Toolchain.Abstractions.Sources; +namespace OceanApocalypse.RSML.Abstractions.Sources; /// /// Represents a sequential scanner. diff --git a/src/RSML.Toolchain.Abstractions/ToolchainConfigurations.cs b/src/RSML.Abstractions/ToolchainConfigurations.cs similarity index 96% rename from src/RSML.Toolchain.Abstractions/ToolchainConfigurations.cs rename to src/RSML.Abstractions/ToolchainConfigurations.cs index 800ed01..871b348 100644 --- a/src/RSML.Toolchain.Abstractions/ToolchainConfigurations.cs +++ b/src/RSML.Abstractions/ToolchainConfigurations.cs @@ -1,7 +1,7 @@ using System; -namespace OceanApocalypse.RSML.Toolchain.Abstractions; +namespace OceanApocalypse.RSML.Abstractions; /// /// Configuration options for a . diff --git a/src/RSML.Language.Parsing/IParser.cs b/src/RSML.Language.Parsing/IParser.cs index a98e140..b5abc56 100644 --- a/src/RSML.Language.Parsing/IParser.cs +++ b/src/RSML.Language.Parsing/IParser.cs @@ -1,4 +1,4 @@ -using OceanApocalypse.RSML.Toolchain.Abstractions; +using OceanApocalypse.RSML.Abstractions; namespace OceanApocalypse.RSML.Language.Parsing; diff --git a/src/RSML.Language.Parsing/Parser.cs b/src/RSML.Language.Parsing/Parser.cs index aab3381..e0803e3 100644 --- a/src/RSML.Language.Parsing/Parser.cs +++ b/src/RSML.Language.Parsing/Parser.cs @@ -1,6 +1,6 @@ using System; -using OceanApocalypse.RSML.Toolchain.Abstractions; +using OceanApocalypse.RSML.Abstractions; namespace OceanApocalypse.RSML.Language.Parsing; diff --git a/src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs b/src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs deleted file mode 100644 index 82aacc3..0000000 --- a/src/RSML.Toolchain.Abstractions/GlobalSuppressions.cs +++ /dev/null @@ -1,8 +0,0 @@ -// This file is used by Code Analysis to maintain SuppressMessage -// attributes that are applied to this project. -// Project-level suppressions either have no target or are given -// a specific target and scoped to a namespace, type, member, etc. - -using System.Diagnostics.CodeAnalysis; - -[assembly: SuppressMessage("Style", "IDE0305:Simplify collection initialization", Justification = "Hides an allocation.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics.DiagnosticCollector.GetAll~System.Collections.Immutable.ImmutableArray{OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics.Diagnostic}")] From 53ce2fd0c82c98b2bbcf9551f490ef162f4dee6f Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 9 Aug 2026 23:42:49 +0100 Subject: [PATCH 06/11] feat(lexer)!: Finish implementing the lexer (yet to test) Refs: #49 --- src/RSML.Language.Lexing/BufferLexer.cs | 72 ++++++++++-- .../Diagnostics/LexerErrorCodes.cs | 2 + .../GlobalSuppressions.cs | 8 ++ src/RSML.Language.Lexing/ILexer.cs | 4 +- src/RSML.Language.Lexing/Lexer.cs | 26 +---- src/RSML.Language.Lexing/ScannerLexer.cs | 4 +- src/RSML.Language.Lexing/Tokens/Token.cs | 104 ++++++++++++++++++ src/RSML.Language.Lexing/Tokens/TokenKind.cs | 78 +++++++++---- 8 files changed, 240 insertions(+), 58 deletions(-) create mode 100644 src/RSML.Language.Lexing/GlobalSuppressions.cs diff --git a/src/RSML.Language.Lexing/BufferLexer.cs b/src/RSML.Language.Lexing/BufferLexer.cs index c92eaa6..3e84554 100644 --- a/src/RSML.Language.Lexing/BufferLexer.cs +++ b/src/RSML.Language.Lexing/BufferLexer.cs @@ -4,9 +4,10 @@ using OceanApocalypse.RSML.Language.Lexing.Diagnostics; using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Toolchain.Abstractions; -using OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; -using OceanApocalypse.RSML.Toolchain.Abstractions.Sources; +using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Diagnostics; +using OceanApocalypse.RSML.Abstractions.Sources; +using OceanApocalypse.RSML.Abstractions.Panic; namespace OceanApocalypse.RSML.Language.Lexing; @@ -49,16 +50,30 @@ public override Result GetNextToken() if (Char.IsAsciiLetter(c) || c == '_') return ScanIdentifierOrKeyword(startLoc); + // standard library identifiers + if (c == '$') + return ScanStdIdentifier(startLoc); + + // member access notation if (c == '.') return Result.Success(new Token(TokenKind.MemberAccess, null, new(startLoc, ++cursor))); - // todo: add the remaining possible paths - return Result.Failure(new(LexerErrorCodes.FailedToLexToken, "Tried all possible token logic paths, but none was true.", Severity.Error)); + // punctuation + if (c.IsAsciiPunctuation()) + return ScanPunctuation(startLoc); + + return Result.Failure(new( + LexerErrorCodes.FailedToLexToken, + "Tried all possible token logic paths, but none was true. This likely means you used a character not recognized by the lexer," + + "but it may also mean the lexer is mal-functioning.", + Severity.Critical + )); } /// public override IEnumerable Lex() { + // todo: make these customizable configurations int maxFailedRunsLimit = 10; int failedRuns = 0; @@ -79,6 +94,10 @@ public override IEnumerable Lex() else yield return token.Value; } + + throw new ExceededMaxAmountOfFailuresException( + $"This instance of the lexer was allowed to fail up to {maxFailedRunsLimit} times, yet it failed {failedRuns}." + ); } private Result ScanNumber(int startLoc) @@ -90,7 +109,7 @@ private Result ScanNumber(int startLoc) if (buffer[cursor] == '.') { if (dot) - return Result.Success(new Token(TokenKind.Number, null, new(startLoc, cursor))); + return Result.Success(new Token(TokenKind.NumericLiteral, null, new(startLoc, cursor))); else dot = true; @@ -99,7 +118,7 @@ private Result ScanNumber(int startLoc) cursor++; } - return Result.Success(new Token(TokenKind.Number, null, new(startLoc, cursor))); + return Result.Success(new Token(TokenKind.NumericLiteral, null, new(startLoc, cursor))); } private Result ScanStringLiteral(int startLoc) @@ -135,6 +154,25 @@ private Result ScanStringLiteral(int startLoc) return Result.Success(new Token(TokenKind.StringLiteral, null, startLoc..cursor)); } + private Result ScanStdIdentifier(int startLoc) + { + // this points to h in $helloWorld broski + int afterStdSymbolIndex = ++cursor; // we also skip past it to avoid extra checks in while loop + + while (cursor < buffer.Length && (Char.IsAsciiLetterOrDigit(buffer[cursor]) || buffer[cursor] == '_')) + cursor++; + + return cursor == afterStdSymbolIndex + ? Result.Failure(new( + LexerErrorCodes.ExpectedStdIdentifier, + buffer.GetLocationDetails((Index)startLoc), + buffer.GetLocationDetails((Index)cursor), + "Expected a standard library identifier, yet there was no valid identifier after the $ symbol.", + Severity.Error + )) + : Result.Success(new Token(TokenKind.StandardLibraryIdentifier, null, startLoc..cursor)); + } + private Result ScanIdentifierOrKeyword(int startLoc) { while (cursor < buffer.Length && (Char.IsAsciiLetterOrDigit(buffer[cursor]) || buffer[cursor] == '_')) @@ -144,7 +182,7 @@ private Result ScanIdentifierOrKeyword(int startLoc) if (Keywords.Contains(buffer[range])) { - var token = new Token(GetKeywordTokenKind(buffer[range]), null, range); // is keyword + var token = new Token(Token.GetKeywordKind(buffer[range]), null, range); // is keyword return token.Kind == TokenKind.Unknown ? Result.Failure(new( @@ -163,6 +201,24 @@ private Result ScanIdentifierOrKeyword(int startLoc) } } + private Result ScanPunctuation(int startLoc) + { + char c = buffer[cursor]; + char? peeked = cursor + 1 >= buffer.Length ? null : buffer[++cursor]; // dont error out if out of bounds + TokenKind kind = Token.GetPunctuationKind(c, peeked); + + return kind == TokenKind.Unknown + ? Result.Failure(new( + LexerErrorCodes.FailedToIdentifyPunctuation, + buffer.GetLocationDetails(startLoc), + buffer.GetLocationDetails(peeked is null ? cursor - 1 : cursor), + "Despite identifying the object in question as punctuation, the lexer failed to resolve exactly which punctuation it was." + + "This might mean the punctuation in question is reserved for future use, and not implemented yet.", + Severity.Error + )) + : Result.Success(new Token(kind, null, startLoc..cursor)); + } + private void SkipWhitespaceAndComments() { while (cursor < buffer.Length) diff --git a/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs b/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs index 700a8fc..b87355e 100644 --- a/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs +++ b/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs @@ -6,4 +6,6 @@ internal static class LexerErrorCodes public const string FailedToLexToken = "RL0001"; public const string UnterminatedStringLiteral = "RL0002"; public const string FailedToIdentifyKeyword = "RL0003"; + public const string FailedToIdentifyPunctuation = "RL0004"; + public const string ExpectedStdIdentifier = "RL0005"; } diff --git a/src/RSML.Language.Lexing/GlobalSuppressions.cs b/src/RSML.Language.Lexing/GlobalSuppressions.cs new file mode 100644 index 0000000..960b92c --- /dev/null +++ b/src/RSML.Language.Lexing/GlobalSuppressions.cs @@ -0,0 +1,8 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("Style", "IDE0046:Convert to conditional expression", Justification = "Would make the code ternary hell.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Language.Lexing.BufferLexer.GetNextToken~OceanApocalypse.RSML.Abstractions.Diagnostics.Result{OceanApocalypse.RSML.Language.Lexing.Tokens.Token}")] diff --git a/src/RSML.Language.Lexing/ILexer.cs b/src/RSML.Language.Lexing/ILexer.cs index 36405ee..4c551ff 100644 --- a/src/RSML.Language.Lexing/ILexer.cs +++ b/src/RSML.Language.Lexing/ILexer.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Toolchain.Abstractions; -using OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; +using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Diagnostics; namespace OceanApocalypse.RSML.Language.Lexing; diff --git a/src/RSML.Language.Lexing/Lexer.cs b/src/RSML.Language.Lexing/Lexer.cs index 5787b92..46888ae 100644 --- a/src/RSML.Language.Lexing/Lexer.cs +++ b/src/RSML.Language.Lexing/Lexer.cs @@ -3,8 +3,8 @@ using System.Collections.Immutable; using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Toolchain.Abstractions; -using OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; +using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Diagnostics; namespace OceanApocalypse.RSML.Language.Lexing; @@ -26,28 +26,6 @@ public abstract class Lexer : ILexer "class", "interface" ]; - // this should always be synced with the keywords field - internal static TokenKind GetKeywordTokenKind(scoped ReadOnlySpan keyword) => keyword switch - { - // keywords - "as" => TokenKind.As, - "end" => TokenKind.End, - "if" => TokenKind.If, - "let" => TokenKind.Let, - "region" => TokenKind.Region, - "requires" => TokenKind.Requires, - "return" => TokenKind.Return, - "struct" => TokenKind.Struct, - "type" => TokenKind.Type, - - // modifiers - "fn" => TokenKind.FunctionModifier, - "mut" => TokenKind.MutableModifier, - "previous" => TokenKind.PreviousModifier, - - _ => TokenKind.Unknown, - }; - private bool isDisposed; /// diff --git a/src/RSML.Language.Lexing/ScannerLexer.cs b/src/RSML.Language.Lexing/ScannerLexer.cs index ae05517..2efce8c 100644 --- a/src/RSML.Language.Lexing/ScannerLexer.cs +++ b/src/RSML.Language.Lexing/ScannerLexer.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Toolchain.Abstractions.Diagnostics; -using OceanApocalypse.RSML.Toolchain.Abstractions.Sources; +using OceanApocalypse.RSML.Abstractions.Diagnostics; +using OceanApocalypse.RSML.Abstractions.Sources; namespace OceanApocalypse.RSML.Language.Lexing; diff --git a/src/RSML.Language.Lexing/Tokens/Token.cs b/src/RSML.Language.Lexing/Tokens/Token.cs index ec16a92..4235ccb 100644 --- a/src/RSML.Language.Lexing/Tokens/Token.cs +++ b/src/RSML.Language.Lexing/Tokens/Token.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; namespace OceanApocalypse.RSML.Language.Lexing.Tokens; @@ -15,4 +16,107 @@ public record struct Token(TokenKind Kind, object? Value, Range Range) /// Empty token. Used when something goes wrong. /// public readonly static Token Empty = new(TokenKind.Unknown, null, new()); + + /// + /// Gets the token kind that applies to the given keyword or keyword modifier. + /// + /// The keyword or modifier. + /// The matching token kind. + public static TokenKind GetKeywordKind(scoped ReadOnlySpan keyword) => keyword switch + { + // keywords + "as" => TokenKind.AsKeyword, + "end" => TokenKind.EndKeyword, + "if" => TokenKind.IfKeyword, + "let" => TokenKind.LetKeyword, + "region" => TokenKind.RegionKeyword, + "requires" => TokenKind.RequiresKeyword, + "return" => TokenKind.ReturnKeyword, + "struct" => TokenKind.StructKeyword, + "type" => TokenKind.TypeKeyword, + + // modifiers + "fn" => TokenKind.FunctionModifier, + "mut" => TokenKind.MutableModifier, + "previous" => TokenKind.PreviousModifier, + + _ => TokenKind.Unknown, + }; + + /// + /// Gets the token kind that applies to the given punctuation symbol. + /// + /// The punctuation character. + /// + /// The character that follows . + /// Set to null if out of bounds. Default is null. + /// + /// + public static TokenKind GetPunctuationKind(char punctuation, char? peekedChar = null) => punctuation switch + { + // math operations + '+' => TokenKind.Plus, + '-' => TokenKind.Minus, + '*' => TokenKind.Star, + '/' => TokenKind.Slash, + + // equality + '=' when peekedChar is '=' => TokenKind.EqualToOperator, + '!' when peekedChar is '=' => TokenKind.NotEqualToOperator, + '>' when peekedChar is '=' => TokenKind.GreaterThanOrEqualToOperator, + '<' when peekedChar is '=' => TokenKind.LessThanOrEqualToOperator, + '>' => TokenKind.GreaterThanOperator, + '<' => TokenKind.LessThanOperator, + + // boolean logic + '&' when peekedChar is '&' => TokenKind.LogicAndOperator, + '|' when peekedChar is '|' => TokenKind.LogicOrOperator, + '!' => TokenKind.NotOperator, + + '=' => TokenKind.AssignmentOperator, + + // reserved + '&' => TokenKind.Unknown, + '|' => TokenKind.Unknown, + + _ => TokenKind.Unknown + }; + + /// + /// Returns true if the given token kind matches a valid keyword that is not a modifier. + /// + /// The token kind to check against. + /// True if the kind is a keyword. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsStrictlyKeyword(TokenKind kind) => + IsKeywordOrModifier(kind) && !IsStrictlyKeywordModifier(kind); + + /// + /// Returns true if the given token kind matches a valid keyword modifier. + /// + /// The token kind to check against. + /// True if the kind is a keyword modifier. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsStrictlyKeywordModifier(TokenKind kind) => + kind is TokenKind.PreviousModifier or TokenKind.MutableModifier or TokenKind.FunctionModifier; + + /// + /// Returns true if the given token kind points to an identifier, be it + /// from the standard library or not. + /// + /// The token kind to check against. + /// True if the kind is an identifier. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsIdentifier(TokenKind kind) => + kind is TokenKind.Identifier or TokenKind.StandardLibraryIdentifier; + + /// + /// Returns true if the given token kind points to any keyword, be it + /// a keyword or a keyword modifier. + /// + /// The token kind to check against. + /// True if the kind is a keyword or modifier. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsKeywordOrModifier(TokenKind kind) => + kind is >= TokenKind.ReturnKeyword and <= TokenKind.StructKeyword; } diff --git a/src/RSML.Language.Lexing/Tokens/TokenKind.cs b/src/RSML.Language.Lexing/Tokens/TokenKind.cs index a5d99c1..9eef629 100644 --- a/src/RSML.Language.Lexing/Tokens/TokenKind.cs +++ b/src/RSML.Language.Lexing/Tokens/TokenKind.cs @@ -8,6 +8,11 @@ public enum TokenKind /// /// An unknown token kind. /// + /// + /// :::note + /// Unknown tokens usually lead to toolchain errors. + /// ::: + /// Unknown, /// @@ -23,7 +28,7 @@ public enum TokenKind /// /// A numeric literal. /// - Number, + NumericLiteral, /// /// A string literal. @@ -31,29 +36,29 @@ public enum TokenKind StringLiteral, /// - /// A built-in constant. + /// A built-in identifier. /// - SystemConstant, + StandardLibraryIdentifier, /// /// The return keyword. Stops execution of the current scope with a given value. /// - Return, + ReturnKeyword, /// /// The if keyword. Conditionalizes a statement into running only if the condition is met. /// - If, + IfKeyword, /// /// The requires keyword. Indicates extensions the file depends on. /// - Requires, + RequiresKeyword, /// /// The end keyword. Ends the file. /// - End, + EndKeyword, /// /// The previous keyword. Modifies end into closing the previous region instead. @@ -63,12 +68,12 @@ public enum TokenKind /// /// The region keyword. Creates a conditionalized region. /// - Region, + RegionKeyword, /// /// The let keyword. Declares and assigns a constant. /// - Let, + LetKeyword, /// /// The mut keyword. Modifies let into creating a variable instead. @@ -80,55 +85,61 @@ public enum TokenKind /// FunctionModifier, + /// + /// The exec keyword. Executes a function without you having to use discards. + /// Treats every function as a void function. + /// + ExecKeyword, + /// /// The type keyword. Creates a type. /// - Type, + TypeKeyword, /// /// The as keyword. /// - As, + AsKeyword, /// /// The struct keyword. Used with type and as to create a struct type. /// - Struct, + StructKeyword, /// /// The assignment operator (=). /// - Assignment, + AssignmentOperator, /// /// The equality operator (==). /// - Equality, + EqualToOperator, /// /// The inequality operator (!=). /// - Inequality, + NotEqualToOperator, /// /// The greater-than operator (>). /// - GreaterThan, + GreaterThanOperator, /// /// The less-than operator (<). /// - LessThan, + LessThanOperator, /// /// The greater-than-or-equal-to operator (>=). /// - GreaterThanOrEqualTo, + GreaterThanOrEqualToOperator, /// /// The less-than-or-equal-to operator (<=). /// - LessThanOrEqualTo, + LessThanOrEqualToOperator, /// /// The colon (:). @@ -173,7 +184,7 @@ public enum TokenKind /// /// The closed brace (}). /// - CloseBrace, + ClosedBrace, /// /// The open parenthesis. @@ -183,10 +194,33 @@ public enum TokenKind /// /// The closed parenthesis. /// - CloseParenthesis, + ClosedParenthesis, /// /// The member access mark (.), which is a dot. /// - MemberAccess + MemberAccess, + + /// + /// The NOT operator. It swaps the boolean value of whatever + /// comes next. + /// + NotOperator, + + /// + /// The logic AND operator. Returns true only if both the + /// left and right sides evaluate to true. + /// + LogicAndOperator, + + /// + /// The logic OR operator. Returns true if either left, right + /// or both sides evaluate to true. + /// + LogicOrOperator, + + /// + /// The at symbol (@). Reserved for future use. + /// + AtSymbol } From a2915cd9dfed69f3ab55aac928feb806d6d3c170 Mon Sep 17 00:00:00 2001 From: Matthew Date: Mon, 10 Aug 2026 23:42:38 +0100 Subject: [PATCH 07/11] refactor(buffer)!: Officially remove the custom buffers and scanners They were heavy to maintain and didn't add much, as the lexer barely used the APIs they exposed. On top of that, .NET has many better ways to handle things like that. Currently code is fundamentally broken, so CI will fail. --- RedSeaModernLanguage.slnx | 12 +- benchmarks/Program.cs | 4 +- benchmarks/RSML.Benchmarks.csproj | 1 - benchmarks/Sources/BufferBenchmarks.cs | 63 - .../AssemblyInfo.cs | 1 - .../Diagnostic.cs | 10 +- .../AssemblyInfo.cs | 1 + .../AssemblyInfo.cs | 15 + .../IToolchainComponent.cs | 7 +- .../RSML.Abstractions.Toolchain.csproj} | 12 +- .../ToolchainConfiguration.cs | 77 + src/RSML.Abstractions/AssemblyInfo.cs | 1 - src/RSML.Abstractions/Extensions.cs | 57 +- src/RSML.Abstractions/Sources/IBuffer.cs | 193 --- src/RSML.Abstractions/Sources/IScanner.cs | 27 - .../ToolchainConfigurations.cs | 59 - src/RSML.Language.Lexing/AssemblyInfo.cs | 1 - src/RSML.Language.Lexing/BufferLexer.cs | 243 ---- .../GlobalSuppressions.cs | 2 +- src/RSML.Language.Lexing/Lexer.cs | 63 - .../RSML.Language.Lexing.csproj | 1 + src/RSML.Language.Lexing/ScannerLexer.cs | 21 - .../RSML.Language.Parsing.csproj | 1 + ...L.Toolchain.Extensibility.Execution.csproj | 1 + .../GlobalSuppressions.cs | 9 - .../ReadOnlyStringBuffer.cs | 766 ---------- tests/RSML.Tests/RSML.Tests.csproj | 1 - .../Sources/ReadOnlyStringBufferTests.cs | 1269 ----------------- 28 files changed, 169 insertions(+), 2749 deletions(-) delete mode 100644 benchmarks/Sources/BufferBenchmarks.cs rename src/{RSML.Toolchain.Sources => RSML.Abstractions.Panic}/AssemblyInfo.cs (94%) create mode 100644 src/RSML.Abstractions.Toolchain/AssemblyInfo.cs rename src/{RSML.Abstractions => RSML.Abstractions.Toolchain}/IToolchainComponent.cs (67%) rename src/{RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj => RSML.Abstractions.Toolchain/RSML.Abstractions.Toolchain.csproj} (57%) create mode 100644 src/RSML.Abstractions.Toolchain/ToolchainConfiguration.cs delete mode 100644 src/RSML.Abstractions/Sources/IBuffer.cs delete mode 100644 src/RSML.Abstractions/Sources/IScanner.cs delete mode 100644 src/RSML.Abstractions/ToolchainConfigurations.cs delete mode 100644 src/RSML.Language.Lexing/BufferLexer.cs delete mode 100644 src/RSML.Language.Lexing/Lexer.cs delete mode 100644 src/RSML.Language.Lexing/ScannerLexer.cs delete mode 100644 src/RSML.Toolchain.Sources/GlobalSuppressions.cs delete mode 100644 src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs delete mode 100644 tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs diff --git a/RedSeaModernLanguage.slnx b/RedSeaModernLanguage.slnx index ac86c85..12ea3dc 100644 --- a/RedSeaModernLanguage.slnx +++ b/RedSeaModernLanguage.slnx @@ -43,6 +43,12 @@ + + + + + + @@ -103,12 +109,6 @@ - - - - - - diff --git a/benchmarks/Program.cs b/benchmarks/Program.cs index 0a74bdf..a84de00 100644 --- a/benchmarks/Program.cs +++ b/benchmarks/Program.cs @@ -1,10 +1,8 @@ using BenchmarkDotNet.Running; -using OceanApocalypse.RSML.Benchmarks.Sources; - namespace OceanApocalypse.RSML.Benchmarks; internal sealed class Program { - private static void Main(string[] args) => BenchmarkRunner.Run(args: args); + private static void Main(string[] args) { } // todo: => BenchmarkRunner.Run(args: args); } diff --git a/benchmarks/RSML.Benchmarks.csproj b/benchmarks/RSML.Benchmarks.csproj index 55ceccb..ae9ff59 100644 --- a/benchmarks/RSML.Benchmarks.csproj +++ b/benchmarks/RSML.Benchmarks.csproj @@ -17,6 +17,5 @@ - \ No newline at end of file diff --git a/benchmarks/Sources/BufferBenchmarks.cs b/benchmarks/Sources/BufferBenchmarks.cs deleted file mode 100644 index ec1509e..0000000 --- a/benchmarks/Sources/BufferBenchmarks.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; - -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Jobs; - -using OceanApocalypse.RSML.Benchmarks.Helpers; -using OceanApocalypse.RSML.Toolchain.Sources; - - -namespace OceanApocalypse.RSML.Benchmarks.Sources; - -[SimpleJob(RuntimeMoniker.Net10_0)] -[SimpleJob(RuntimeMoniker.Net80)] -[SimpleJob(RuntimeMoniker.NativeAot10_0)] -[SimpleJob(RuntimeMoniker.NativeAot80)] -[MemoryDiagnoser] -[CsvExporter] -[RPlotExporter] -[SuppressMessage("Maintainability", "CA1515:Consider making public types internal", Justification = "Benchmarks have to be public.")] -public class BufferBenchmarks : IDisposable -{ - private bool isDisposed; - private readonly Consumer consumer = new(); - private string data = ""; - private ReadOnlyStringBuffer buffer = null!; - - [Params(1, 10, 100, 1_000)] - public int RepeatCount { get; set; } // 1 is the string itself - - [GlobalSetup] - public void Setup() - { - data = DataGenerator.GetSampleData(RepeatCount); - buffer = new(data); - } - - [Benchmark] - public void ReadOnlyStringBuffer_GetLine() - { - for (int i = 0; i <= data.Length; i++) - consumer.Consume(buffer.GetLineNumberFromIndex(i)); - } - - [GlobalCleanup] - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) - { - if (isDisposed) - return; - - if (disposing) // managed resources - buffer.Dispose(); - - isDisposed = true; - } -} diff --git a/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs b/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs index 3fa1539..26fce2e 100644 --- a/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs +++ b/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs @@ -13,4 +13,3 @@ [assembly: ComVisible(false)] [assembly: CLSCompliant(true)] -[assembly: InternalsVisibleTo("RSML.Toolchain.Sources")] diff --git a/src/RSML.Abstractions.Diagnostics/Diagnostic.cs b/src/RSML.Abstractions.Diagnostics/Diagnostic.cs index cffb462..aee4332 100644 --- a/src/RSML.Abstractions.Diagnostics/Diagnostic.cs +++ b/src/RSML.Abstractions.Diagnostics/Diagnostic.cs @@ -43,7 +43,7 @@ namespace OceanApocalypse.RSML.Abstractions.Diagnostics; /// The error code. public Diagnostic(string code) { - ArgumentNullException.ThrowIfNullOrWhiteSpace(code); + ArgumentException.ThrowIfNullOrWhiteSpace(code); ThrowIfInvalidErrorCode(code); Code = code; @@ -56,7 +56,7 @@ public Diagnostic(string code) /// A brief error message detailing why it has happened. public Diagnostic(string code, string message) { - ArgumentNullException.ThrowIfNullOrWhiteSpace(code); + ArgumentException.ThrowIfNullOrWhiteSpace(code); ThrowIfInvalidErrorCode(code); Code = code; @@ -69,7 +69,7 @@ public Diagnostic(string code, string message) /// The error's severity. public Diagnostic(string code, Severity severity) { - ArgumentNullException.ThrowIfNullOrWhiteSpace(code); + ArgumentException.ThrowIfNullOrWhiteSpace(code); ThrowIfInvalidErrorCode(code); Code = code; @@ -83,7 +83,7 @@ public Diagnostic(string code, Severity severity) /// The error's severity. public Diagnostic(string code, string message, Severity severity) { - ArgumentNullException.ThrowIfNullOrWhiteSpace(code); + ArgumentException.ThrowIfNullOrWhiteSpace(code); ThrowIfInvalidErrorCode(code); Code = code; @@ -99,7 +99,7 @@ public Diagnostic(string code, string message, Severity severity) /// The error's severity. public Diagnostic(string code, (Index idx, int line, int col) spanStart, (Index idx, int line, int col) spanEnd, string message, Severity severity) { - ArgumentNullException.ThrowIfNullOrWhiteSpace(code); + ArgumentException.ThrowIfNullOrWhiteSpace(code); ThrowIfInvalidErrorCode(code); Code = code; diff --git a/src/RSML.Toolchain.Sources/AssemblyInfo.cs b/src/RSML.Abstractions.Panic/AssemblyInfo.cs similarity index 94% rename from src/RSML.Toolchain.Sources/AssemblyInfo.cs rename to src/RSML.Abstractions.Panic/AssemblyInfo.cs index 1b4601d..26fce2e 100644 --- a/src/RSML.Toolchain.Sources/AssemblyInfo.cs +++ b/src/RSML.Abstractions.Panic/AssemblyInfo.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // In SDK-style projects such as this one, several assembly attributes that were historically diff --git a/src/RSML.Abstractions.Toolchain/AssemblyInfo.cs b/src/RSML.Abstractions.Toolchain/AssemblyInfo.cs new file mode 100644 index 0000000..26fce2e --- /dev/null +++ b/src/RSML.Abstractions.Toolchain/AssemblyInfo.cs @@ -0,0 +1,15 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// In SDK-style projects such as this one, several assembly attributes that were historically +// defined in this file are now automatically added during build and populated with +// values defined in project properties. For details of which attributes are included +// and how to customise this process see: https://aka.ms/assembly-info-properties + +// Setting ComVisible to false makes the types in this assembly not visible to COM +// components. If you need to access a type in this assembly from COM, set the ComVisible +// attribute to true on that type. +[assembly: ComVisible(false)] + +[assembly: CLSCompliant(true)] diff --git a/src/RSML.Abstractions/IToolchainComponent.cs b/src/RSML.Abstractions.Toolchain/IToolchainComponent.cs similarity index 67% rename from src/RSML.Abstractions/IToolchainComponent.cs rename to src/RSML.Abstractions.Toolchain/IToolchainComponent.cs index d9f1dad..b99968d 100644 --- a/src/RSML.Abstractions/IToolchainComponent.cs +++ b/src/RSML.Abstractions.Toolchain/IToolchainComponent.cs @@ -1,6 +1,7 @@ using System; +using System.Collections.Generic; -namespace OceanApocalypse.RSML.Abstractions; +namespace OceanApocalypse.RSML.Abstractions.Toolchain; /// /// A component of the RSML toolchain. @@ -10,11 +11,11 @@ public interface IToolchainComponent : IDisposable /// /// Configurations for the toolchain component. /// - ToolchainConfigurations Configuration { get; } + ToolchainConfiguration Configuration { get; } /// /// Injects a configuration into the toolchain component, modifying it. /// /// The configuration to inject. - void Inject(ToolchainConfigurations configuration); + void Inject(ToolchainConfiguration configuration); } diff --git a/src/RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj b/src/RSML.Abstractions.Toolchain/RSML.Abstractions.Toolchain.csproj similarity index 57% rename from src/RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj rename to src/RSML.Abstractions.Toolchain/RSML.Abstractions.Toolchain.csproj index 09854eb..f7e94d4 100644 --- a/src/RSML.Toolchain.Sources/RSML.Toolchain.Sources.csproj +++ b/src/RSML.Abstractions.Toolchain/RSML.Abstractions.Toolchain.csproj @@ -1,19 +1,19 @@ - true Library - OceanApocalypse.RSML.Toolchain.Sources + OceanApocalypse.RSML.Abstractions.Toolchain True - OceanApocalypse.RSML.Toolchain.Sources - RSML Sources + OceanApocalypse.RSML.Abstractions.Toolchain + RSML Toolchain Abstractions - + - + \ No newline at end of file diff --git a/src/RSML.Abstractions.Toolchain/ToolchainConfiguration.cs b/src/RSML.Abstractions.Toolchain/ToolchainConfiguration.cs new file mode 100644 index 0000000..b89e631 --- /dev/null +++ b/src/RSML.Abstractions.Toolchain/ToolchainConfiguration.cs @@ -0,0 +1,77 @@ +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Data; +using System.Runtime.CompilerServices; + +namespace OceanApocalypse.RSML.Abstractions.Toolchain; + +/// +/// Contains configurations for the entire toolchain. +/// +public record ToolchainConfiguration() +{ + private bool isFrozen; + private readonly Dictionary custom = []; + + /// + /// The maximum amount of diagnostics needed for the toolchain to stop, per component.
+ /// 0 does not limit failures.
+ /// 1 simulates a fast fail: it's only recommended for CI purposes. + /// + public int MaximumAllowedFailuresPerComponent + { + get; + set + { + ThrowIfFrozen(); + field = value; + } + } = 100; + + /// + /// Whether or not the lexer should emit comment tokens. + /// Setting to false is only recommended if only interpreting (no analysis tools). + /// + public bool EmitComments + { + get; + set + { + ThrowIfFrozen(); + field = value; + } + } = true; + + /// + /// The custom configurations. + /// + public IReadOnlyDictionary CustomConfigurations => custom.ToFrozenDictionary(); + + /// + /// The default toolchain configuration. + /// + public static ToolchainConfiguration Default { get; } = new(); + + /// + /// Sets a custom configuration. + /// + /// The configuration's key. + /// The configuration's value. + public void SetCustom(string key, string value) + { + ThrowIfFrozen(); + custom[key] = value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ThrowIfFrozen() + { + if (isFrozen) + throw new ReadOnlyException("Configurations are frozen and cannot be changed."); + } + + /// + /// Freezes the configurations, preventing any future mutations. + /// + public void Freeze() => isFrozen = true; +} \ No newline at end of file diff --git a/src/RSML.Abstractions/AssemblyInfo.cs b/src/RSML.Abstractions/AssemblyInfo.cs index 3fa1539..26fce2e 100644 --- a/src/RSML.Abstractions/AssemblyInfo.cs +++ b/src/RSML.Abstractions/AssemblyInfo.cs @@ -13,4 +13,3 @@ [assembly: ComVisible(false)] [assembly: CLSCompliant(true)] -[assembly: InternalsVisibleTo("RSML.Toolchain.Sources")] diff --git a/src/RSML.Abstractions/Extensions.cs b/src/RSML.Abstractions/Extensions.cs index e85290c..f28c75a 100644 --- a/src/RSML.Abstractions/Extensions.cs +++ b/src/RSML.Abstractions/Extensions.cs @@ -9,22 +9,65 @@ namespace OceanApocalypse.RSML.Abstractions; /// public static class Extensions { - extension(char character) + private const byte UpperLowerDiffBit = 0b_0010_0000; // 0x20, binary seems best suited for this tho ngl + + #region ASCII Characters + private const byte Tab = 0x9; + private const byte Lf = 0xA; + private const byte Cr = 0xD; + private const byte Space = 0x20; + private const byte Exclamation = 0x21; + private const byte And = 0x26; + private const byte LessThan = 0x3C; + private const byte GreaterThan = 0x3E; + private const byte UppercaseA = 0x41; + private const byte UppercaseZ = 0x5A; + private const byte Pipe = 0x7C; + #endregion + + extension(byte item) { /// - /// Checks if the character in question represents a newline. Allowed newlines are: - /// CR, LF, line break and paragraph break. + /// Checks if the character in question represents an ASCII newline. /// - /// + /// True if ASCII newline. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsNewline() => character is '\r' or '\n' or '\u2028' or '\u2029'; + public bool IsAsciiNewline() => item is Lf or Cr; /// /// Checks if the character in question is ASCII punctuation. Used by RSML. /// - /// + /// True if ASCII punctuation. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsRsmlPunctuation() => item is (>= LessThan and <= GreaterThan) or Exclamation or And or Pipe; + + /// + /// Checks if the character in question is ASCII whitespace. + /// + /// True if ASCII punctuation. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsAsciiWhitespace() => item is Space or (>= Tab and <= Cr); + + /// + /// Converts an ASCII letter to its uppercase form. + /// + /// The uppercase ASCII letter. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public byte ToAsciiUpperInvariant() => (byte)(item & (~UpperLowerDiffBit)); + + /// + /// Checks if a given character is an ASCII letter. + /// + /// True if the character is an ASCII letter. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsAsciiLetter() => ToAsciiUpperInvariant(item) is >= UppercaseA and <= UppercaseZ; + + /// + /// Checks if a given character is an ASCII digit. + /// + /// True if the character is an ASCII digit. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsAsciiPunctuation() => character is '=' or '<' or '>' or '!' or '|' or '&'; + public bool IsAsciiDigit() => item is >= 48 and <= 57; // 48 is '0' and 57 is '9' } extension(IImmutableList strings) diff --git a/src/RSML.Abstractions/Sources/IBuffer.cs b/src/RSML.Abstractions/Sources/IBuffer.cs deleted file mode 100644 index b72c845..0000000 --- a/src/RSML.Abstractions/Sources/IBuffer.cs +++ /dev/null @@ -1,193 +0,0 @@ -using System; - -namespace OceanApocalypse.RSML.Abstractions.Sources; - -/// -/// Represents a buffer of characters. -/// -public interface IBuffer : IDisposable, IEquatable, IEquatable, IEquatable, IEquatable> -{ - // todo: add methods that can mutate the buffer (coming to v3.0.0-prerelease2 ??) - - /// - /// Whether the source is completely empty. - /// - bool IsEmpty => Length == 0; - - /// - /// Whether the source can be mutated. - /// - bool IsReadOnly { get; } - - /// - /// The length of the source. - /// - int Length { get; } - - /// - /// The total amount of lines in the buffer. - /// - /// - /// Keep in mind lines might be empty. - /// - int LineCount { get; } - - /// - /// Gets a single item out of the buffer. - /// - /// The index of the item to retrieve. - /// The item. - char this[int index] { get; } - - /// - /// Gets a single item out of the buffer. - /// - /// The index of the item to retrieve. - /// The item. - char this[Index index] { get; } - - /// - /// Gets a span of items out of the buffer. - /// - /// The range to retrieve. - /// The items. - ReadOnlySpan this[Range range] { get; } - - /// - /// Counts the amount of items until the next line separator in the buffer, relative to a given . - /// Only line separators count - regular whitespace do not. CRLF counts as a single line separator, to avoid double counting. - /// - /// The index at which to start counting. - /// - /// Whether the line separator at which the method stopped is the CR in a CRLF sequence. If true, the next item in the buffer is LF. - /// - /// The index of the next line separator, relative to an . - int CountUntilEndOfLine(Index index, out bool isCrLf); - - /// - /// Counts the amount of items until the next non-whitespace item in the buffer, relative to a given . - /// Line separators are included in the whitespace category. - /// - /// The index at which to start counting. - /// The index of the next non-whitespace item, relative to a . - int CountUntilNotWhitespace(Index index); - - /// - /// Counts the amount of items until the next whitespace item in the buffer, relative to a given . - /// Line separators are included in the whitespace category. - /// - /// The index at which to start counting. - /// The index of the next whitespace item, relative to a . - int CountUntilWhitespace(Index index); - - /// - /// Counts the amount of items, starting from a given , - /// while a returns true. - /// - /// - /// The index at which to start counting; all indexes will also be given to the - /// as an offset that when added to the index of the position - /// equal the actual index. - /// - /// - /// A function that takes the current index (relative to ), - /// which is incremented every item, and the item associated with it. Execution stops when - /// the predicate returns false or the index is out of bounds. - /// - /// The amount of items counted. - int CountWhile(Index index, Func predicate); - - /// - /// Returns the length of a line given its 0-based line number. - /// Line separators do not count towards the length. - /// - /// The 0-based line number. - /// The length of the line. - int GetLengthOfLine(int lineNumber); - - /// - /// Returns the length of a line given a 0-based index of one - /// of its items. - /// Line separators do not count towards the length. - /// - /// The 0-based index whose line is considered. - /// The length of the line. - int GetLengthOfLineFromIndex(Index index); - - /// - /// Given a 0-based line number, returns the matching line as an array of buffer items. - /// - /// The 0-based line number. - /// The line as an array of items. - ReadOnlySpan GetLine(int lineNumber); - - /// - /// Tries to read the line that contains the item at . - /// No end of line characters are added. - /// - /// The index at which to determine what the current line is. - /// The line, as an array of items. - ReadOnlySpan GetLineFromIndex(Index index); - - /// - /// Determines the 0-based line number of the line that contains the item located at . - /// - /// The index whose parent line's number is to be returned. - /// The 0-based number of the line that contains item located at . - int GetLineNumberFromIndex(Index index); - - /// - /// Converts an index into a location. - /// - /// The index. - /// The location. - (Index Index, int Line, int Column) GetLocationDetails(Index index); - - /// - /// Slices a region of the buffer. - /// - /// The index of the first item in the slice. - /// The amount of items to slice starting at . - /// A slice, as an array of items. - ReadOnlySpan Slice(Index start, int length); - - /// - /// Slices a region of the buffer into a performant span. - /// - /// The index of the first item in the slice. - /// The span serving as the destination for the slice. - bool TrySlice(Index start, Span slice); - - /// - /// Slices a region of the buffer into a performant span. - /// - /// The range indicating what the slice is. - /// The span serving as the destination for the slice. - bool TrySlice(Range range, Span slice); - - /// - /// Tries to return the item at the specified . - /// - /// The item's location. - /// The item. - /// False if the buffer is out of bounds or an exception occured. - bool TryGetChar(Index index, out char item); - - /// - /// Given a 0-based line number, assigns the exact line to a result buffer (). - /// No end of line characters are added. - /// - /// The 0-based line number. - /// The destination buffer for the line. - /// True if successful. - bool TryGetLine(int lineNumber, Span destination); - - /// - /// Tries to read the line that contains the item at . - /// No end of line characters are added. - /// - /// The index at which to determine what the current line is. - /// The destination span that will contain the line. - /// True if successful. - bool TryGetLineFromIndex(Index index, Span destination); -} diff --git a/src/RSML.Abstractions/Sources/IScanner.cs b/src/RSML.Abstractions/Sources/IScanner.cs deleted file mode 100644 index a71b742..0000000 --- a/src/RSML.Abstractions/Sources/IScanner.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; - - -namespace OceanApocalypse.RSML.Abstractions.Sources; - -/// -/// Represents a sequential scanner. -/// -public interface IScanner : IDisposable, IEquatable -{ - // todo: planned for v3.0.0-prerelease2 - - /// - /// The current index of the cursor. - /// - int CursorIndex { get; } - - /// - /// Whether the source is completely empty. - /// - bool IsEmpty { get; } - - /// - /// Whether the source can be mutated. - /// - bool IsReadOnly { get; } -} diff --git a/src/RSML.Abstractions/ToolchainConfigurations.cs b/src/RSML.Abstractions/ToolchainConfigurations.cs deleted file mode 100644 index 871b348..0000000 --- a/src/RSML.Abstractions/ToolchainConfigurations.cs +++ /dev/null @@ -1,59 +0,0 @@ -using System; - - -namespace OceanApocalypse.RSML.Abstractions; - -/// -/// Configuration options for a . -/// -[Flags] -public enum ToolchainConfigurations -{ - /// - /// Optimizes the toolchain pipeline by disabling extension processing. - /// - /// - /// :::warning - /// This completely disables extensions, but does not warn you if there are active extensions, - /// meaning sometimes you might be wondering why your extension is not working when, in reality, - /// you've enabled this configuration. - /// ::: - /// - /// :::tip - /// This configuration is automatically enabled when no extensions are enabled. - /// ::: - /// - DisableExtensionProcessing = 1, - - /// - /// Only allows OceanApocalypse extensions, leading to an error if any non-OAS extension is active. - /// - /// - /// :::note - /// When used alongside , the non-OAS extensions will be disabled, but - /// no errors will be thrown. - /// ::: - /// - AllowOnlyOASExtensions = 2, - - /// - /// Ignores all errors caused by broken or faulty extensions. - /// - IgnoreBrokenExtensions = 4, - - /// - /// Ignores all errors caused by injecting already injected extensions. - /// - IgnoreDuplicatedExtensions = 8, - - /// - /// Ignores all errors thrown during pipeline creation and pipeline execution. - /// - /// - /// :::danger - /// This option is only needed in beyond extremely rare occasions. - /// It emulates RSML v1.x.x behavior. - /// ::: - /// - IgnoreAllExtensibilityErrors = IgnoreBrokenExtensions | IgnoreDuplicatedExtensions -} diff --git a/src/RSML.Language.Lexing/AssemblyInfo.cs b/src/RSML.Language.Lexing/AssemblyInfo.cs index 5017d62..26fce2e 100644 --- a/src/RSML.Language.Lexing/AssemblyInfo.cs +++ b/src/RSML.Language.Lexing/AssemblyInfo.cs @@ -13,4 +13,3 @@ [assembly: ComVisible(false)] [assembly: CLSCompliant(true)] -[assembly: InternalsVisibleTo("RSML.Toolchain.Extensibility.Lexing")] diff --git a/src/RSML.Language.Lexing/BufferLexer.cs b/src/RSML.Language.Lexing/BufferLexer.cs deleted file mode 100644 index 3e84554..0000000 --- a/src/RSML.Language.Lexing/BufferLexer.cs +++ /dev/null @@ -1,243 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; - -using OceanApocalypse.RSML.Language.Lexing.Diagnostics; -using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Abstractions; -using OceanApocalypse.RSML.Abstractions.Diagnostics; -using OceanApocalypse.RSML.Abstractions.Sources; -using OceanApocalypse.RSML.Abstractions.Panic; - -namespace OceanApocalypse.RSML.Language.Lexing; - -/// -/// An implementation of a RSML lexer backed by a read-only or read-and-write buffer. -/// -/// A buffer. Can be read-only () or read and write (). -/// A collector for all emitted diagnostics. -public class BufferLexer(IBuffer buffer, DiagnosticCollector diagnostics) : Lexer -{ - private int cursor; - - /// - /// :::note[Diagnostic output] - /// This method does not add diagnostics to the collector - /// (): it only returns them when it - /// proves necessary. - /// ::: - /// - /// - public override Result GetNextToken() - { - SkipWhitespaceAndComments(); - - if (cursor >= buffer.Length) - return Result.Success(new Token(TokenKind.Eof, null, new())); - - int startLoc = cursor; - char c = buffer[cursor]; - - // strings - if (c == '"') - return ScanStringLiteral(startLoc); - - // number literals - if (Char.IsAsciiDigit(c)) - return ScanNumber(startLoc); - - // identifiers and keywords - if (Char.IsAsciiLetter(c) || c == '_') - return ScanIdentifierOrKeyword(startLoc); - - // standard library identifiers - if (c == '$') - return ScanStdIdentifier(startLoc); - - // member access notation - if (c == '.') - return Result.Success(new Token(TokenKind.MemberAccess, null, new(startLoc, ++cursor))); - - // punctuation - if (c.IsAsciiPunctuation()) - return ScanPunctuation(startLoc); - - return Result.Failure(new( - LexerErrorCodes.FailedToLexToken, - "Tried all possible token logic paths, but none was true. This likely means you used a character not recognized by the lexer," + - "but it may also mean the lexer is mal-functioning.", - Severity.Critical - )); - } - - /// - public override IEnumerable Lex() - { - // todo: make these customizable configurations - int maxFailedRunsLimit = 10; - int failedRuns = 0; - - while (failedRuns < maxFailedRunsLimit) - { - var token = GetNextToken(); - - if (token.IsError) - { - diagnostics.Add(token.Error); - failedRuns++; - continue; - } - - if (token.Value.Kind == TokenKind.Eof) - yield break; - - else - yield return token.Value; - } - - throw new ExceededMaxAmountOfFailuresException( - $"This instance of the lexer was allowed to fail up to {maxFailedRunsLimit} times, yet it failed {failedRuns}." - ); - } - - private Result ScanNumber(int startLoc) - { - bool dot = false; - - while (cursor < buffer.Length && (Char.IsAsciiDigit(buffer[cursor]) || buffer[cursor] == '_' || buffer[cursor] == '.')) - { - if (buffer[cursor] == '.') - { - if (dot) - return Result.Success(new Token(TokenKind.NumericLiteral, null, new(startLoc, cursor))); - - else - dot = true; - } - - cursor++; - } - - return Result.Success(new Token(TokenKind.NumericLiteral, null, new(startLoc, cursor))); - } - - private Result ScanStringLiteral(int startLoc) - { - cursor++; - bool escaping = false; - - while (cursor < buffer.Length) - { - if (buffer[cursor].IsNewline()) - { - return Result.Failure(new( - LexerErrorCodes.UnterminatedStringLiteral, - buffer.GetLocationDetails((Index)startLoc), - buffer.GetLocationDetails((Index)cursor), - "A string literal must begin and end in the same line.", - Severity.Error - )); - } - - if (buffer[cursor] == '"' && !escaping) - break; - - if (buffer[cursor] == '\\') - escaping = !escaping; - - cursor++; - } - - if (cursor < buffer.Length) - cursor++; // skip end quote if anything beyond it - - return Result.Success(new Token(TokenKind.StringLiteral, null, startLoc..cursor)); - } - - private Result ScanStdIdentifier(int startLoc) - { - // this points to h in $helloWorld broski - int afterStdSymbolIndex = ++cursor; // we also skip past it to avoid extra checks in while loop - - while (cursor < buffer.Length && (Char.IsAsciiLetterOrDigit(buffer[cursor]) || buffer[cursor] == '_')) - cursor++; - - return cursor == afterStdSymbolIndex - ? Result.Failure(new( - LexerErrorCodes.ExpectedStdIdentifier, - buffer.GetLocationDetails((Index)startLoc), - buffer.GetLocationDetails((Index)cursor), - "Expected a standard library identifier, yet there was no valid identifier after the $ symbol.", - Severity.Error - )) - : Result.Success(new Token(TokenKind.StandardLibraryIdentifier, null, startLoc..cursor)); - } - - private Result ScanIdentifierOrKeyword(int startLoc) - { - while (cursor < buffer.Length && (Char.IsAsciiLetterOrDigit(buffer[cursor]) || buffer[cursor] == '_')) - cursor++; - - Range range = startLoc..cursor; - - if (Keywords.Contains(buffer[range])) - { - var token = new Token(Token.GetKeywordKind(buffer[range]), null, range); // is keyword - - return token.Kind == TokenKind.Unknown - ? Result.Failure(new( - LexerErrorCodes.FailedToIdentifyKeyword, - buffer.GetLocationDetails(startLoc), - buffer.GetLocationDetails(cursor), - "Despite identifying the object in question as a keyword, the lexer failed to resolve exactly which keyword it was." + - "This likely means the keyword in question is reserved for future use, but isn't implemented yet.", - Severity.Error - )) - : Result.Success(token); - } - else - { - return Result.Success(new Token(TokenKind.Identifier, null, range)); // is identifier - } - } - - private Result ScanPunctuation(int startLoc) - { - char c = buffer[cursor]; - char? peeked = cursor + 1 >= buffer.Length ? null : buffer[++cursor]; // dont error out if out of bounds - TokenKind kind = Token.GetPunctuationKind(c, peeked); - - return kind == TokenKind.Unknown - ? Result.Failure(new( - LexerErrorCodes.FailedToIdentifyPunctuation, - buffer.GetLocationDetails(startLoc), - buffer.GetLocationDetails(peeked is null ? cursor - 1 : cursor), - "Despite identifying the object in question as punctuation, the lexer failed to resolve exactly which punctuation it was." + - "This might mean the punctuation in question is reserved for future use, and not implemented yet.", - Severity.Error - )) - : Result.Success(new Token(kind, null, startLoc..cursor)); - } - - private void SkipWhitespaceAndComments() - { - while (cursor < buffer.Length) - { - char c = buffer[cursor]; - - if (Char.IsWhiteSpace(c)) - { - cursor += buffer.CountUntilNotWhitespace(cursor); - } - else if (c == '#') - { - while (!buffer[cursor].IsNewline()) - cursor++; - } - else - { - break; - } - } - } -} diff --git a/src/RSML.Language.Lexing/GlobalSuppressions.cs b/src/RSML.Language.Lexing/GlobalSuppressions.cs index 960b92c..382fe03 100644 --- a/src/RSML.Language.Lexing/GlobalSuppressions.cs +++ b/src/RSML.Language.Lexing/GlobalSuppressions.cs @@ -5,4 +5,4 @@ using System.Diagnostics.CodeAnalysis; -[assembly: SuppressMessage("Style", "IDE0046:Convert to conditional expression", Justification = "Would make the code ternary hell.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Language.Lexing.BufferLexer.GetNextToken~OceanApocalypse.RSML.Abstractions.Diagnostics.Result{OceanApocalypse.RSML.Language.Lexing.Tokens.Token}")] +[assembly: SuppressMessage("Style", "IDE0046:Convert to conditional expression", Justification = "Would make the code ternary hell.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Language.Lexing.Utf8Lexer.GetNextToken(System.Buffers.SequenceReader{System.Byte}@)~OceanApocalypse.RSML.Abstractions.Diagnostics.Result{OceanApocalypse.RSML.Language.Lexing.Tokens.Token}")] diff --git a/src/RSML.Language.Lexing/Lexer.cs b/src/RSML.Language.Lexing/Lexer.cs deleted file mode 100644 index 46888ae..0000000 --- a/src/RSML.Language.Lexing/Lexer.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; - -using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Abstractions; -using OceanApocalypse.RSML.Abstractions.Diagnostics; - -namespace OceanApocalypse.RSML.Language.Lexing; - -/// -/// The base class for implementations of RSML lexers and tokenizers. -/// -public abstract class Lexer : ILexer -{ - /// - /// The RSML keywords defined in its language specification. - /// Also contains reserved keywords. - /// - public static readonly ImmutableArray Keywords = [ - // keywords - "as", "end", "if", "let", "region", "requires", "return", "struct", "type", - // modifiers - "fn", "mut", "previous", - // reserved keywords - not yet implemented but blocked from being used as identifiers - "class", "interface" - ]; - - private bool isDisposed; - - /// - public virtual ToolchainConfigurations Configuration { get; protected set; } - - /// - public virtual void Inject(ToolchainConfigurations configuration) => Configuration |= configuration; - - /// - public abstract IEnumerable Lex(); - - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - /// - /// Disposes of both managed and unmanaged resources. - /// - /// When set to false, disposes of unmanaged resources only. - protected virtual void Dispose(bool disposing) - { - if (isDisposed) - return; - - // dispose of managed stuff if disposing is true - - isDisposed = true; - } - - /// - public abstract Result GetNextToken(); -} diff --git a/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj b/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj index 18ab05f..ca9166f 100644 --- a/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj +++ b/src/RSML.Language.Lexing/RSML.Language.Lexing.csproj @@ -13,6 +13,7 @@ + diff --git a/src/RSML.Language.Lexing/ScannerLexer.cs b/src/RSML.Language.Lexing/ScannerLexer.cs deleted file mode 100644 index 2efce8c..0000000 --- a/src/RSML.Language.Lexing/ScannerLexer.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Collections.Generic; - -using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Abstractions.Diagnostics; -using OceanApocalypse.RSML.Abstractions.Sources; - -namespace OceanApocalypse.RSML.Language.Lexing; - -/// -/// An implementation of a RSML lexer backed by a scanner. -/// -/// A scanner. -/// A collector for all emitted diagnostics. -public class ScannerLexer(IScanner scanner, DiagnosticCollector diagnostics) : Lexer -{ - /// - public override Result GetNextToken() => throw new System.NotImplementedException(); - - /// - public override IEnumerable Lex() => throw new System.NotImplementedException(); -} diff --git a/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj b/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj index a303b29..9b53ea1 100644 --- a/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj +++ b/src/RSML.Language.Parsing/RSML.Language.Parsing.csproj @@ -13,6 +13,7 @@ + diff --git a/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj b/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj index 6167081..1fc8abf 100644 --- a/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj +++ b/src/RSML.Toolchain.Extensibility.Execution/RSML.Toolchain.Extensibility.Execution.csproj @@ -12,6 +12,7 @@ + diff --git a/src/RSML.Toolchain.Sources/GlobalSuppressions.cs b/src/RSML.Toolchain.Sources/GlobalSuppressions.cs deleted file mode 100644 index d76fa6b..0000000 --- a/src/RSML.Toolchain.Sources/GlobalSuppressions.cs +++ /dev/null @@ -1,9 +0,0 @@ -// This file is used by Code Analysis to maintain SuppressMessage -// attributes that are applied to this project. -// Project-level suppressions either have no target or are given -// a specific target and scoped to a namespace, type, member, etc. - -using System.Diagnostics.CodeAnalysis; - -[assembly: SuppressMessage("CodeQuality", "IDE0079:Remove unnecessary suppression", Justification = "Not an unnecessary suppression.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Toolchain.Sources.ReadOnlyStringBuffer.#ctor(System.Byte*,System.Int32,System.Text.Encoding)")] -[assembly: SuppressMessage("CodeQuality", "IDE0079:Remove unnecessary suppression", Justification = "Not an unnecessary suppression.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Toolchain.Sources.ReadOnlyStringBuffer.BuildCache")] diff --git a/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs b/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs deleted file mode 100644 index 50afe4c..0000000 --- a/src/RSML.Toolchain.Sources/ReadOnlyStringBuffer.cs +++ /dev/null @@ -1,766 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; -using System.Text; - -using OceanApocalypse.RSML.Abstractions; -using OceanApocalypse.RSML.Abstractions.Cache; -using OceanApocalypse.RSML.Abstractions.Panic; -using OceanApocalypse.RSML.Abstractions.Sources; - - -namespace OceanApocalypse.RSML.Toolchain.Sources; - -/// -/// A read-only buffer backed by a string. All operations opt for performance -/// primarily via the internal use of over string allocations -/// and also via caching. -/// -public sealed class ReadOnlyStringBuffer : IBuffer, ISupportsCache -{ - private const int AverageCharactersPerLine = 40; - private const int ExtraCharacterCapacity = 64; - private bool isDisposed; - - private readonly List lineStarts = []; - - private readonly List precededByCrLf = []; - - private readonly string data; - - /// - public bool CacheExists { get; private set; } - - /// - public bool IsEmpty => Length == 0; - - /// - /// Always returns true, as only - /// supports read-only content (hence the name). - /// - /// - public bool IsReadOnly => true; - - /// - public int Length => data.Length; - - /// - /// automatically builds cache if - /// no cached data exists. No calls - /// are necessary. - /// - /// - public int LineCount => RawLineCount - 1; // ignore the "fake" EOF line (convention) - - private int RawLineCount - { - get - { - ComputeLineStarts(); - - return lineStarts.Count; - } - } - - /// - public ReadOnlySpan this[Range range] => data.AsSpan()[range]; - - /// - public char this[Index index] => data[index]; - - /// - public char this[int index] => data[index]; - - /// - /// Initializes a new - /// with a string. - /// - /// The string that the buffer will wrap. - public ReadOnlyStringBuffer(string content) => data = content; - - /// - /// Initializes a new - /// by allocating a string from a . - /// - /// The span pointing to the string's data. - public ReadOnlyStringBuffer(ReadOnlySpan content) => data = content.ToString(); - - /// - /// Initializes a new - /// with an array of characters. - /// - /// The array of characters to use for the buffer. - public ReadOnlyStringBuffer(char[] content) => data = new(content); - - /// - /// Initializes a new - /// with an array of bytes and the encoding to use when decoding them. - /// - /// The array of bytes to use for the buffer. - /// - /// The encoding to use when decoding . - /// Use null for the encoding. - /// - public ReadOnlyStringBuffer(byte[] content, Encoding? encoding = null) => data = encoding?.GetString(content) ?? Encoding.Default.GetString(content); - - /// - /// Initializes a new - /// with a pointer referencing an array of bytes and the encoding - /// to use when decoding them. - /// - /// The pointer referecing the array of bytes to use for the buffer. - /// The amount of bytes in the array referenced by . - /// - /// The encoding to use when decoding . - /// Use null for the encoding. - /// - /// This method is not CLS-compliant due to the unsafe context and the use of pointers. - [CLSCompliant(false)] - public unsafe ReadOnlyStringBuffer(byte* contentPtr, int byteCount, Encoding? encoding = null) => - data = (encoding ?? Encoding.Default).GetString(contentPtr, byteCount); - - /// - public void BuildCache() => ComputeLineStarts(); - - /// - public void BuildCache(bool forceRebuild) => ComputeLineStarts(forceRebuild); - - /// - /// :::info[EOF Conventions] - /// This method allows the EOF index as in-range. The convention is as follows: - /// - If the index is EOF (), then the output is always 0 and is always false. - /// - If the index is the last ( - 1), then the output is always 0. - /// ::: - /// - /// :::info[Value of 'isCrLf' parameter] - /// is only true if all the following conditions are true: - /// - The next line start counting from is preceded by a CRLF sequence. - /// - does not point to the LF in the CRLF sequence. - /// - does not point to EOF. - /// ::: - /// - /// - public int CountUntilEndOfLine(Index index, out bool isCrLf) - { - isCrLf = false; - int offset = index.GetOffset(Length); - - ThrowIfEmpty(); - ThrowIfOutOfRange(offset, true); - - if (offset == Length) - return 0; // consumed the entire buffer - - ComputeLineStarts(); - - int lineSep = GetNextLineStartPosition(offset, out _); - isCrLf = precededByCrLf.Contains(lineSep) && data[index] is not '\n'; // to us, CRLF is only when we're not standing on the LF - - if (isCrLf) - lineSep--; // skip the extra line separator in the CRLF sequence - - if (!(IsLastLine(offset) && !data[^1].IsNewline())) // if we're not on the last line and it doesn't end with a newline then - { - lineSep--; - } - - return lineSep - offset; - } - - /// - /// :::info[EOF Conventions] - /// This method allows the EOF index as in-range. - /// If the index is EOF (), then the output is always 0. - /// ::: - /// - /// :::tip[About the return value] - /// The return value, when summed with , becomes the index of the first character that - /// is not whitespace, counting from . - /// The only exception is if the buffer has been consumed (you pass EOF index or there's no more characters that are - /// not whitespace), meaning the return value, when summed with is the value of - /// , which is also the EOF index. - /// ::: - /// - /// - public int CountUntilNotWhitespace(Index index) - { - ThrowIfEmpty(); - int offset = index.GetOffset(Length); - ThrowIfOutOfRange(offset, true); - - if (offset == Length) - return 0; // consumed the entire buffer - - var span = data.AsSpan(index); - int count = 0; - - while (count < span.Length && Char.IsWhiteSpace(span[count])) - count++; - - return count; - } - - /// - /// :::info[EOF Conventions] - /// This method allows the EOF index as in-range. - /// If the index is EOF (), then the output is always 0. - /// ::: - /// - /// :::tip[About the return value] - /// The return value, when summed with , becomes the index of the first character that - /// is whitespace, counting from . - /// The only exception is if the buffer has been consumed (you pass EOF index or there's no more characters that are - /// whitespace), meaning the return value, when summed with is the value of - /// , which is also the EOF index. - /// ::: - /// - /// - public int CountUntilWhitespace(Index index) - { - ThrowIfEmpty(); - int offset = index.GetOffset(Length); - ThrowIfOutOfRange(offset, true); - - if (offset == Length) - return 0; // consumed the entire buffer - - var span = data.AsSpan(offset); - int count = 0; - - while (count < span.Length && !Char.IsWhiteSpace(span[count])) - count++; - - return count; - } - - /// - /// :::info[EOF Conventions] - /// This method allows the EOF index as in-range. - /// If the index is EOF (), then the output is always 0. - /// ::: - /// - /// :::tip[About the return value] - /// The return value, when summed with , becomes the index of the first character that - /// fails to verify the , counting from . - /// The only exception is if the buffer has been consumed (you pass EOF index or there's no more characters that fail to verify - /// the ), meaning the return value, when summed with is the value of - /// , which is also the EOF index. - /// ::: - /// - /// - public int CountWhile(Index index, Func predicate) - { - if (predicate is null) - throw new ArgumentNullException(nameof(predicate), "The object is null."); - - ThrowIfEmpty(); - int offset = index.GetOffset(Length); - ThrowIfOutOfRange(offset, true); - - if (offset == Length) - return 0; // consumed the entire buffer - - var span = data.AsSpan(offset); - int count = 0; - - while (count < span.Length && predicate(count, span[count])) - count++; - - return count; - } - - /// - /// :::info[EOF Conventions] - /// This method follows EOF conventions. - /// EOF is considered a 0-character sequence in line N, where N is . - /// Keep in mind N does not point to an actual line (it's just a convention), as line numbers are 0-based - /// (meaning the actual last line is located at N - 1). - /// ::: - /// - /// - public int GetLengthOfLine(int lineNumber) - { - ThrowIfEmpty(); - ComputeLineStarts(); - ThrowIfLineNumberOutOfRange(lineNumber); - - if (lineNumber == RawLineCount - 1) - return 0; // EOF means the line is empty - - int start = lineStarts[lineNumber]; - int end = lineStarts[lineNumber + 1]; - - if (precededByCrLf.Contains(end)) - end--; // skip the extra line separator in the CRLF sequence - - if (!(lineNumber + 2 == RawLineCount && !data[^1].IsNewline())) - { - // if we're not on the last line and it doesn't end with newline then - end--; // skip one more line separator - } - - return end - start; - } - - /// - /// :::info[EOF Conventions] - /// This method follows EOF conventions. - /// EOF is considered a 0-character sequence in line N, where N is . - /// Keep in mind N does not point to an actual line (it's just a convention), as line numbers are 0-based - /// (meaning the actual last line is located at N - 1). - /// ::: - /// - /// - public int GetLengthOfLineFromIndex(Index index) => GetLengthOfLine(GetLineNumberFromIndex(index)); - - /// - /// :::info[EOF Conventions] - /// This method follows EOF conventions. - /// ::: - /// - /// - public int GetLineNumberFromIndex(Index index) - { - ThrowIfEmpty(); - int offset = index.GetOffset(Length); - ThrowIfOutOfRange(offset, true); - ComputeLineStarts(); - - int lineSepIndex = GetPreviousOrCurrentLineStartPositionInLineStartList(offset); - return lineSepIndex; - } - - /// - /// :::info[EOF Conventions] - /// This method follows EOF conventions. - /// EOF is considered a 0-character sequence in line N, where N is . - /// Keep in mind N does not point to an actual line (it's just a convention), as line numbers are 0-based - /// (meaning the actual last line is located at N - 1). - /// ::: - /// - /// - public ReadOnlySpan GetLine(int lineNumber) - { - ThrowIfEmpty(); - ComputeLineStarts(); - ThrowIfLineNumberOutOfRange(lineNumber); - - if (lineNumber + 1 == RawLineCount) - return String.Empty; // EOF means the line is empty - - int start = lineStarts[lineNumber]; - int length = GetLengthOfLine(lineNumber); - - return data.AsSpan(start, length); - } - - /// - public ReadOnlySpan GetLineFromIndex(Index index) => GetLine(GetLineNumberFromIndex(index)); - - /// - /// :::warning[EOF Conventions] - /// Unlike with other methods, this one - /// does not follow EOF conventions and, because of that, does not accept the - /// EOF index (index at ), because it is not - /// considered part of any slice. - /// ::: - /// - /// - public ReadOnlySpan Slice(Index start, int length) - { - if (length < 0) - throw new ArgumentOutOfRangeException(nameof(length), "The slice length must be positive."); - - int offset = start.GetOffset(Length); - ThrowIfOutOfRange(offset, true, nameof(start)); - - return data.AsSpan(offset, length); - } - - /// - /// :::warning[EOF Conventions] - /// Unlike with other methods, this one - /// does not follow EOF conventions and, because of that, does not accept the - /// EOF index (index at ), because it is not - /// considered part of any slice. - /// ::: - /// - /// - public ReadOnlySpan Slice(Range range) - { - int startOffset = range.Start.GetOffset(Length); - int endOffset = range.End.GetOffset(Length); - - ThrowIfOutOfRange(startOffset, true, nameof(range)); - ThrowIfOutOfRange(endOffset, true, nameof(range)); - - return data.AsSpan()[range]; - } - - /// - /// :::warning[EOF Conventions] - /// Unlike with other methods, this one - /// does not follow EOF conventions and, because of that, does not accept the - /// EOF index (index at ), because it is not - /// considered part of any slice. - /// ::: - /// - /// - public bool TrySlice(Index start, Span slice) => data.AsSpan(start.GetOffset(Length), slice.Length).TryCopyTo(slice); - - /// - public bool TrySlice(Range range, Span slice) => data.AsSpan()[range].TryCopyTo(slice); - - /// - /// :::info[EOF Conventions] - /// This method follows the EOF convention where the EOF character - /// is 0 ('\0') and the return value is false, due to EOF - /// not being an actual buffer location. - /// ::: - /// - /// - public bool TryGetChar(Index index, out char item) - { - item = '\0'; // default - - if (IsEmpty || IsOutOfRange(index.IsFromEnd ? Length + (-index.Value) : index.Value)) - return false; - - item = data[index]; - - return true; - } - - /// - /// :::info[EOF Conventions] - /// This method follows EOF conventions. - /// EOF is considered a 0-character sequence in line N, where N is . - /// Keep in mind N does not point to an actual line (it's just a convention), as line numbers are 0-based - /// (meaning the actual last line is located at N - 1). - /// ::: - /// - /// - public bool TryGetLine(int lineNumber, Span destination) - { - ComputeLineStarts(); - - if (IsEmpty || lineNumber < 0 || lineNumber >= RawLineCount) - return false; - - if (lineNumber + 1 == RawLineCount) - return true; // EOF means the line is empty (and destination is by default empty) - - int start = lineStarts[lineNumber]; - int length = GetLengthOfLine(lineNumber); - - return data.AsSpan(start, length).TryCopyTo(destination); - } - - /// - /// :::info[EOF Conventions] - /// This method follows EOF conventions. - /// EOF is considered a 0-character sequence in line N, where N is . - /// Keep in mind N does not point to an actual line (it's just a convention), as line numbers are 0-based - /// (meaning the actual last line is located at N - 1). If is EOF, the - /// line will also be EOF. - /// ::: - /// - /// - public bool TryGetLineFromIndex(Index index, Span destination) - { - if (IsEmpty || IsOutOfRange(index.IsFromEnd ? Length + (-index.Value) : index.Value, followEofConvention: true)) // avoids panic from GetLineNumberFromIndex - return false; - - var lineNumber = GetLineNumberFromIndex(index); - return TryGetLine(lineNumber, destination); - } - - /// - /// :::warning[EOF Conventions] - /// Unlike with other methods, this one - /// does not follow EOF conventions and, because of that, does not accept the - /// EOF index (index at ), because it is not - /// considered a location. - /// ::: - /// - /// - public (Index Index, int Line, int Column) GetLocationDetails(Index index) - { - ThrowIfEmpty(); - ThrowIfOutOfRange(index.IsFromEnd ? Length + (-index.Value) : index.Value); - - if (index.Value == 0) // best "best" case = triple zero - return (0, 0, 0); - - ComputeLineStarts(); - int lineNumber = GetLineNumberFromIndex(index); - - return new(index, lineNumber, index.GetOffset(Length) - lineStarts[lineNumber]); - } - - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - /// - public override bool Equals( - [NotNullWhen(true)] - object? obj - ) => obj switch - { - string str => Equals(str), - char[] charArray => Equals(charArray), - IBuffer buffer => Equals(buffer), - ReadOnlyMemory readOnlyMemory => Equals(readOnlyMemory), - null => false, - _ => false - }; - - /// - /// Checks if an array of characters is equal to the current instance. - /// - /// The array. - /// True if equals. - public bool Equals(char[]? other) => other is not null && data.Equals(other.AsSpan(), StringComparison.Ordinal); - - /// - /// Checks if another read-only buffer is equal to the current instance. - /// - /// The other read-only buffer. - /// True if equals. - public bool Equals(IBuffer? other) => other is not null && data.Equals(other.ToString(), StringComparison.Ordinal); - - /// - /// Checks if a read-only contiguous region of memory is equal to the current instance. - /// - /// The region of memory. - /// True if equals. - public bool Equals(ReadOnlyMemory other) => Length == other.Length && data.SequenceEqual(other.Span); - - /// - /// Checks if a read-only contiguous region of memory is equal to the current instance. - /// - /// The region of memory. - /// True if equals. - public bool Equals(ReadOnlySpan other) => Length == other.Length && data.SequenceEqual(other); - - /// - /// Checks if a string is equal to the current instance. - /// - /// The string. - /// True if equals. - public bool Equals(string? other) => other is not null && Length == other.Length && data.Equals(other, StringComparison.Ordinal); - - /// - public override int GetHashCode() => unchecked(HashCode.Combine(data, lineStarts, precededByCrLf)); - - /// - /// Returns the buffer's content as a . - /// - /// The buffer's content. - public override string ToString() => data; - - /// - /// Checks if two read-only string buffers are equals. - /// - /// True if equals. - public static bool operator ==(ReadOnlyStringBuffer left, ReadOnlyStringBuffer right) => - EqualityComparer.Default.Equals(left, right); - - /// - /// Checks if two read-only string buffers are different. - /// - /// True if different. - public static bool operator !=(ReadOnlyStringBuffer left, ReadOnlyStringBuffer right) => !(left == right); - - /// - /// Disposes of both managed and unmanaged resources. - /// - /// When set to false, disposes of unmanaged resources only. - private void Dispose(bool disposing) - { - if (isDisposed) - return; - - if (disposing) - { - lineStarts.Clear(); - precededByCrLf.Clear(); - CacheExists = false; - } - - isDisposed = true; - } - - private void ComputeLineStarts(bool forceCache = false) - { - if (CacheExists && !forceCache) - return; - - var span = data.AsSpan(); - - lineStarts.Clear(); - precededByCrLf.Clear(); - - lineStarts.Capacity = Math.Max(lineStarts.Capacity, span.Length / AverageCharactersPerLine + ExtraCharacterCapacity); // just a rough guess - precededByCrLf.Capacity = Math.Max(precededByCrLf.Capacity, span.Length / AverageCharactersPerLine + (OperatingSystem.IsWindows() ? ExtraCharacterCapacity : 0)); - - /* the following line ensures that if the last line: - * ends with CR, LF, U2028 or U2029 - * ends with CRLF - * does not end with any of the above - * - * it will be counted as a line no matter the outcome of the previous condition - * this makes it more obvious from a human side like "bruv my string is abc\ndef two lines right" - * it looks normal that there are 2 lines but someone will go "acshua'y, that's erm 1 line :skull:" - * not with RSML's official buffers nah bro - */ - int lastIndex = span.EndsWith("\r\n") ? span.Length - 2 : span.Length - 1; - int i = 0; - - lineStarts.Add(0); // 0 is by convention the start of line (and also the start of the buffer) - - while (i < lastIndex) - { - if (!span[i].IsNewline()) - { - i++; - continue; - } - - bool isCrLf = i < lastIndex && span[i] == '\r' && span[i + 1] == '\n'; - int nextStart = i + (isCrLf ? 2 : 1); - lineStarts.Add(nextStart); - - if (isCrLf) - precededByCrLf.Add(nextStart); - - i = nextStart; - } - - lineStarts.Add(Length); // add the EOF as the start of a line - - CacheExists = true; - } - - /// The insertion point of the next line start. - /// The index, in where the line start is. - /// The line start index, in . - private int GetNextLineStartFromInsertionPoint(int insertionPoint, out int lsListIndex) - { - if (insertionPoint >= 0) - { - // 1st Case: the used index (might have been index + 1 based on the method that called this) - // points to an actual line start - lsListIndex = insertionPoint; - return lineStarts[insertionPoint]; - } - - int nextIndex = ~insertionPoint; - - if (nextIndex == lineStarts.Count) - { - // 2nd Case: the next line start is outside of the buffer (EOF convention) - lsListIndex = nextIndex; - return Length; - } - - lsListIndex = nextIndex; - - return lineStarts[nextIndex]; // 3rd Case: we found the next line start - } - - private int GetNextLineStartPosition(int index, out int lsListIndex) => - // we use index + 1 to skip to the next line start if we're already standing on one :) - GetNextLineStartFromInsertionPoint(lineStarts.BinarySearch(index + 1), out lsListIndex); - - /// The insertion point of the next line start. - /// The line start index in , in . - private int GetPreviousLineStartInLineStartListFromInsertionPoint(int insertionPoint) - { - if (insertionPoint >= 0) - { - // 1st Case: the used index (might have been index + 1 based on the method that called this) - // points to an actual line start - return insertionPoint; - } - - int previousIndex = ~insertionPoint; - - if (previousIndex == lineStarts.Count) - { - // 2nd Case: the next line start is outside of the buffer (EOF convention) - // however we're gonna decrement one because otherwise all indexes from the last line that are after - // start of said line suddenly become part of the EOF line - // however this might also mean we're at last character - return previousIndex; - } - - return previousIndex == 0 ? 0 : previousIndex - 1; // 3rd Case: we found the previous line start - } - - private int GetPreviousOrCurrentLineStartPositionInLineStartList(int index) - { - if (index > 0) - { - // when the index is greater than 0, we can safely get the previous line start without getting a big shitty error - // we use 0 as the start following standard conventions - return GetPreviousLineStartInLineStartListFromInsertionPoint(lineStarts.BinarySearch(index)); - } - - return 0; - } - - private bool IsLastLine(int index) => RawLineCount == 1 || index >= lineStarts[^2] && index < lineStarts[^1]; - - /// - /// If is set to 'false': - /// True if index is greater than or equal to the length. - /// This prevents throwing - /// or . - /// If is set to 'true': - /// True if index is greater than the length. - /// This does NOT prevent throwing - /// or . - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsOutOfRange(int index, bool followEofConvention = false) => - index < 0 || index > Length || (!followEofConvention && index == Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int NormalizeIndex(int index) => index < 0 ? index + Length : index; - - private void ThrowIfLineNumberOutOfRange(int lineNumber, string? paramName = null) - { - if (lineNumber < 0 || lineNumber >= RawLineCount) - { - throw new ArgumentOutOfRangeException( - paramName ?? nameof(lineNumber), - "The line number is negative or greather than the buffer's line count, meaning it doesn't point to either any valid character or EOF." - ); - } - } - - private void ThrowIfOutOfRange(int index, bool followEofConvention = false, string? paramName = null) - { - if (IsOutOfRange(index, followEofConvention)) - { - throw new ArgumentOutOfRangeException( - paramName ?? nameof(index), - followEofConvention - ? "The index is negative or greather than the buffer's length, meaning it doesn't point to either any valid character or EOF." - : "The index is negative, greater than or equal to the buffer's length, meaning it doesn't point to any valid character. EOF is not allowed." - ); - } - } - - private void ThrowIfEmpty() - { - if (IsEmpty) - throw new BufferException("panic: The buffer is empty and, therefore, all indexes are out of range."); - } -} diff --git a/tests/RSML.Tests/RSML.Tests.csproj b/tests/RSML.Tests/RSML.Tests.csproj index 8c2d984..2cdddeb 100644 --- a/tests/RSML.Tests/RSML.Tests.csproj +++ b/tests/RSML.Tests/RSML.Tests.csproj @@ -23,7 +23,6 @@ - diff --git a/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs b/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs deleted file mode 100644 index 761a117..0000000 --- a/tests/RSML.Tests/Sources/ReadOnlyStringBufferTests.cs +++ /dev/null @@ -1,1269 +0,0 @@ -using System; -using System.Text; - -using OceanApocalypse.RSML.Abstractions.Panic; -using OceanApocalypse.RSML.Toolchain.Sources; - -namespace OceanApocalypse.RSML.Tests.Sources; - -/// -/// Tests for the official-provided class. -/// -public class ReadOnlyStringBufferTests -{ - private const string TestString01 = "Hey\r\nThis\rIs\u2029A Test \n Method\r\n\r\n.\u2028"; - private const string TestString02 = "Hey\r\nThis\rIs\u2029A Test \n Method\r\n\r\n."; - private const string TestString03 = "This string has a lotofwhitespace charact\u2029\u2028ers out of\r\nnowhere !! "; - private const string TestString04 = "this STRING MIXES a LOT\n of \u2029dIffereNT cas1ngs RAND0mLy!?"; - private const string TestString05 = "!rrrrrrrrr"; - private const string TestString06 = "r!!!!!!!!!"; - private const string TestString07 = "This tests the\nLine\rCount property\r\n"; - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 3, true)] // H in "Hey" - [InlineData(TestString01, 3, 0, true)] // CR in "Hey\r\n" - [InlineData(TestString01, 4, 0, false)] // LF in "Hey\r\n" - [InlineData(TestString01, 5, 4, false)] // T in "This" - [InlineData(TestString01, 6, 3, false)] // h in "This" - [InlineData(TestString01, 13, 7, false)] // A in "A Test" - [InlineData(TestString01, 15, 5, false)] // T in "Test" - [InlineData(TestString01, 22, 6, true)] // M in "Method" - [InlineData(TestString01, 28, 0, true)] // First CR in "\r\n\r\n." - [InlineData(TestString01, 29, 0, false)] // First LF in "\r\n\r\n." - [InlineData(TestString01, 30, 0, true)] // Second CR in "\r\n\r\n." - [InlineData(TestString01, 31, 0, false)] // Second LF in "\r\n\r\n." - [InlineData(TestString01, 32, 1, false)] // Dot/point in "\r\n\r\n." - [InlineData(TestString01, 33, 0, false)] // U2028 in ".\u2028" - [InlineData(TestString01, 34, 0, false)] // End of file - #endregion - #region String ends without newline - [InlineData(TestString02, 0, 3, true)] // H in "Hey" - [InlineData(TestString02, 3, 0, true)] // CR in "Hey\r\n" - [InlineData(TestString02, 4, 0, false)] // LF in "Hey\r\n" - [InlineData(TestString02, 5, 4, false)] // T in "This" - [InlineData(TestString02, 6, 3, false)] // h in "This" - [InlineData(TestString02, 13, 7, false)] // A in "A Test" - [InlineData(TestString02, 15, 5, false)] // T in "Test" - [InlineData(TestString02, 22, 6, true)] // M in "Method" - [InlineData(TestString02, 28, 0, true)] // First CR in "\r\n\r\n." - [InlineData(TestString02, 29, 0, false)] // First LF in "\r\n\r\n." - [InlineData(TestString02, 30, 0, true)] // Second CR in "\r\n\r\n." - [InlineData(TestString02, -2, 0, false)] // Second LF in "\r\n\r\n." - [InlineData(TestString02, -1, 1, false)] // Dot/point in "\r\n\r\n." - [InlineData(TestString02, 33, 0, false)] // End of file - #endregion - #region String with a lot of whitespace - [InlineData(TestString03, 0, 50, false)] - [InlineData(TestString03, 7, 43, false)] - [InlineData(TestString03, 19, 31, false)] - [InlineData(TestString03, 32, 18, false)] - [InlineData(TestString03, 49, 1, false)] - [InlineData(TestString03, 50, 0, false)] - [InlineData(TestString03, 51, 0, false)] - [InlineData(TestString03, 52, 10, true)] - [InlineData(TestString03, 59, 3, true)] - [InlineData(TestString03, 61, 1, true)] - [InlineData(TestString03, 62, 0, true)] - #endregion - #region String with a single line - [InlineData(TestString05, 0, 10, false)] - [InlineData(TestString05, 9, 1, false)] - [InlineData(TestString05, 10, 0, false)] - [InlineData(TestString06, 0, 10, false)] - [InlineData(TestString06, 3, 7, false)] - [InlineData(TestString06, 6, 4, false)] - [InlineData(TestString06, 8, 2, false)] - [InlineData(TestString06, 9, 1, false)] - [InlineData(TestString06, 10, 0, false)] - #endregion - public void CountUntilEndOfLine(string data, int index, int expectedCount, bool expectedCrLf) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountUntilEndOfLine(index < 0 ? new(-index, true) : (Index)index, out bool actualCrLf)); - Assert.Equal(expectedCrLf, actualCrLf); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(2)] - [InlineData(14)] - [InlineData(99)] - [InlineData(-10)] - [InlineData(-1)] - #endregion - public void CountUntilEndOfLine_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - bool isCrLf = true; - Assert.Throws(() => buffer.CountUntilEndOfLine(index < 0 ? new(-index, true) : (Index)index, out isCrLf)); - Assert.False(isCrLf); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - [InlineData(TestString02, -36)] - [InlineData(TestString05, -11)] - #endregion - public void CountUntilEndOfLine_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - bool isCrLf = true; - Assert.Throws(() => buffer.CountUntilEndOfLine(index < 0 ? new(-index, true) : (Index)index, out isCrLf)); - Assert.False(isCrLf); - } - - [Theory] - #region Attributes - [InlineData(TestString03, 0, 0)] // T in "This" - [InlineData(TestString03, 2, 0)] // i in "This" - [InlineData(TestString03, 3, 0)] // s in "This" - [InlineData(TestString03, 4, 2)] // whitespace after "This" - [InlineData(TestString03, 5, 1)] // whitespace after "This" - [InlineData(TestString03, 6, 0)] // s in "string" - [InlineData(TestString03, 9, 0)] // i in "string" - [InlineData(TestString03, 11, 0)] // g in "string" - [InlineData(TestString03, 12, 5)] // whitespace after "string" - [InlineData(TestString03, 14, 3)] // whitespace after "string" - [InlineData(TestString03, 15, 2)] // whitespace after "string" - [InlineData(TestString03, 16, 1)] // whitespace after "string" - [InlineData(TestString03, 20, 1)] // whitespace after "has" - [InlineData(TestString03, 21, 0)] // "a" after "has" - [InlineData(TestString03, 28, 0)] // w in "lotofwhitespace" - [InlineData(TestString03, 38, 5)] // whitespace after "lotofwhitespace" - [InlineData(TestString03, 39, 4)] // whitespace after "lotofwhitespace" - [InlineData(TestString03, 42, 1)] // whitespace after "lotofwhitespace" - [InlineData(TestString03, 46, 0)] // r in "charact\u2029\u2028" - [InlineData(TestString03, 50, 2)] // U2029 in "charact\u2029\u2028" - [InlineData(TestString03, 51, 1)] // U2028 in "charact\u2029\u2028" - [InlineData(TestString03, 52, 0)] // e in "\u2029\u2028ers" - [InlineData(TestString03, 59, 1)] // whitespace after "out" - [InlineData(TestString03, 62, 2)] // CR in "\r\nnowhere" - [InlineData(TestString03, 63, 1)] // LF in "\r\nnowhere" - [InlineData(TestString03, 65, 0)] // o in "nowhere" - [InlineData(TestString03, 69, 0)] // r in "nowhere" - [InlineData(TestString03, 70, 0)] // last e in "nowhere" - [InlineData(TestString03, 71, 1)] // whitespace after "nowhere" - [InlineData(TestString03, 73, 0)] // second exclamation mark - [InlineData(TestString03, 74, 1)] // whitespace after exclamation marks - [InlineData(TestString03, 75, 0)] // End of file - #endregion - public void CountUntilNotWhitespace(string data, int index, int expectedCount) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountUntilNotWhitespace(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(2)] - [InlineData(14)] - [InlineData(99)] - [InlineData(-10)] - #endregion - public void CountUntilNotWhitespace_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.CountUntilNotWhitespace(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - #endregion - public void CountUntilNotWhitespace_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.CountUntilNotWhitespace(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(TestString03, 0, 4)] // T in "This" - [InlineData(TestString03, 2, 2)] // i in "This" - [InlineData(TestString03, 3, 1)] // s in "This" - [InlineData(TestString03, 4, 0)] // whitespace after "This" - [InlineData(TestString03, 5, 0)] // whitespace after "This" - [InlineData(TestString03, 6, 6)] // s in "string" - [InlineData(TestString03, 9, 3)] // i in "string" - [InlineData(TestString03, 11, 1)] // g in "string" - [InlineData(TestString03, 12, 0)] // whitespace after "string" - [InlineData(TestString03, 14, 0)] // whitespace after "string" - [InlineData(TestString03, 15, 0)] // whitespace after "string" - [InlineData(TestString03, 16, 0)] // whitespace after "string" - [InlineData(TestString03, 20, 0)] // whitespace after "has" - [InlineData(TestString03, 21, 1)] // "a" after "has" - [InlineData(TestString03, 28, 10)] // w in "lotofwhitespace" - [InlineData(TestString03, 38, 0)] // whitespace after "lotofwhitespace" - [InlineData(TestString03, 39, 0)] // whitespace after "lotofwhitespace" - [InlineData(TestString03, 42, 0)] // whitespace after "lotofwhitespace" - [InlineData(TestString03, 46, 4)] // r in "charact\u2029\u2028" - [InlineData(TestString03, 50, 0)] // U2029 in "charact\u2029\u2028" - [InlineData(TestString03, 51, 0)] // U2028 in "charact\u2029\u2028" - [InlineData(TestString03, 52, 3)] // e in "\u2029\u2028ers" - [InlineData(TestString03, 59, 0)] // whitespace after "out" - [InlineData(TestString03, 62, 0)] // CR in "\r\nnowhere" - [InlineData(TestString03, 63, 0)] // LF in "\r\nnowhere" - [InlineData(TestString03, 65, 6)] // o in "nowhere" - [InlineData(TestString03, 69, 2)] // r in "nowhere" - [InlineData(TestString03, 70, 1)] // last e in "nowhere" - [InlineData(TestString03, 71, 0)] // whitespace after "nowhere" - [InlineData(TestString03, 73, 1)] // second exclamation mark - [InlineData(TestString03, 74, 0)] // whitespace after exclamation marks - [InlineData(TestString03, 75, 0)] // End of file - #endregion - public void CountUntilWhitespace(string data, int index, int expectedCount) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountUntilWhitespace(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(2)] - [InlineData(14)] - [InlineData(99)] - [InlineData(-10)] - #endregion - public void CountUntilWhitespace_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.CountUntilWhitespace(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - #endregion - public void CountUntilWhitespace_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.CountUntilWhitespace(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(TestString01)] - [InlineData(TestString02)] - [InlineData(TestString03)] - [InlineData(TestString04)] - #endregion - public void CountWhile_SameAsLengthIfAlwaysTrue(string data) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(data.Length, buffer.CountWhile(0, (_, _) => true)); - } - - [Theory] - #region Regular string - [InlineData(TestString03, 0, 8)] // T in "This - [InlineData(TestString03, 2, 6)] // i in "This" - [InlineData(TestString03, 5, 3)] // whitespace before "string" - [InlineData(TestString03, 7, 1)] // t in "This string" - [InlineData(TestString03, 8, 0)] // r in "This string" - [InlineData(TestString03, 9, 37)] // i in "string" - [InlineData(TestString03, 10, 36)] - [InlineData(TestString03, 11, 35)] - [InlineData(TestString03, 15, 31)] - [InlineData(TestString03, 18, 28)] - [InlineData(TestString03, 20, 26)] - [InlineData(TestString03, 22, 24)] - [InlineData(TestString03, 24, 22)] - [InlineData(TestString03, 29, 17)] - [InlineData(TestString03, 31, 15)] - [InlineData(TestString03, 34, 12)] - [InlineData(TestString03, 37, 9)] - [InlineData(TestString03, 41, 5)] - [InlineData(TestString03, 44, 2)] - [InlineData(TestString03, 45, 1)] - [InlineData(TestString03, 46, 0)] - [InlineData(TestString03, 47, 6)] // second in "charact" - [InlineData(TestString03, 48, 5)] - [InlineData(TestString03, 50, 3)] - [InlineData(TestString03, 51, 2)] - [InlineData(TestString03, 53, 0)] - [InlineData(TestString03, 54, 15)] // s in "ers " - [InlineData(TestString03, 55, 14)] - [InlineData(TestString03, 58, 11)] - [InlineData(TestString03, 61, 8)] - [InlineData(TestString03, 63, 6)] - [InlineData(TestString03, 64, 5)] - [InlineData(TestString03, 67, 2)] - [InlineData(TestString03, 68, 1)] - [InlineData(TestString03, 69, 0)] - [InlineData(TestString03, 70, 5)] - [InlineData(TestString03, 72, 3)] // after last r in the whole string - [InlineData(TestString03, 74, 1)] // before End of file - [InlineData(TestString03, 75, 0)] // End of file - #endregion - #region String with several r's - [InlineData(TestString05, 0, 1)] // "!" - [InlineData(TestString05, 1, 0)] - [InlineData(TestString05, 4, 0)] - [InlineData(TestString05, 7, 0)] - [InlineData(TestString05, 9, 0)] - [InlineData(TestString05, 10, 0)] // EOF - #endregion - #region String with a single r - [InlineData(TestString06, 0, 0)] // "r" - [InlineData(TestString06, 1, 9)] - [InlineData(TestString06, 4, 6)] - [InlineData(TestString06, 7, 3)] - [InlineData(TestString06, 9, 1)] - [InlineData(TestString06, 10, 0)] // EOF - #endregion - public void CountWhile_CountsWhileNotLowercaseR(string data, int index, int expectedCount) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, c) => c != 'r')); - } - - [Theory] - #region Attributes - [InlineData(TestString04, 0, 0)] // t in "this STRING" - [InlineData(TestString04, 3, 0)] // s in "this STRING" - [InlineData(TestString04, 4, 14)] // whitespace after "this" - [InlineData(TestString04, 5, 13)] // S in "STRING" - [InlineData(TestString04, 6, 12)] // T in "STRING" - [InlineData(TestString04, 8, 10)] // I in "STRING" - [InlineData(TestString04, 9, 9)] // N in "STRING" - [InlineData(TestString04, 10, 8)] // G in "STRING" - [InlineData(TestString04, 11, 7)] // whitespace after "STRING" - [InlineData(TestString04, 12, 6)] // M in "MIXES" - [InlineData(TestString04, 14, 4)] // X in "MIXES" - [InlineData(TestString04, 16, 2)] // S in "MIXES" - [InlineData(TestString04, 17, 1)] // whitespace after "MIXES" - [InlineData(TestString04, 18, 0)] // "a" surrounded by whitespace - [InlineData(TestString04, 19, 6)] // whitespace after sole "a" - [InlineData(TestString04, 21, 4)] // O in "LOT" - [InlineData(TestString04, 22, 3)] // T in "LOT" - [InlineData(TestString04, 23, 2)] // LF after "LOT" - [InlineData(TestString04, 24, 1)] // whitespace after "LOT\n" - [InlineData(TestString04, 26, 0)] // f in "of" - [InlineData(TestString04, 27, 2)] // whitespace after "of" - [InlineData(TestString04, 28, 1)] // U2029 after "of " - [InlineData(TestString04, 29, 0)] // d in "dIffereNT" - [InlineData(TestString04, 30, 1)] // I in "dIffereNT" - [InlineData(TestString04, 31, 0)] // first f in "dIffereNT" - [InlineData(TestString04, 32, 0)] // second f in "dIffereNT" - [InlineData(TestString04, 35, 0)] // second e in "dIffereNT" - [InlineData(TestString04, 36, 3)] // N in "dIffereNT" - [InlineData(TestString04, 37, 2)] // T in "dIffereNT" - [InlineData(TestString04, 38, 1)] // whitespace after "dIffereNT" - [InlineData(TestString04, 39, 0)] // c in "cas1ngs" - [InlineData(TestString04, 42, 0)] // 1 in "cas1ngs" - [InlineData(TestString04, 46, 5)] // whitespace after "cas1ngs" - [InlineData(TestString04, 49, 2)] // N in "RAND0mLy" - [InlineData(TestString04, 51, 0)] // 0 in "RAND0mLy" - [InlineData(TestString04, 55, 0)] // exclamation mark - [InlineData(TestString04, 56, 0)] // question mark - [InlineData(TestString04, 57, 0)] // End of file - #endregion - public void CountWhile_CountsWhileUppercaseOrWhitespace(string data, int index, int expectedCount) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedCount, buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, c) => Char.IsWhiteSpace(c) || c is '\r' or '\n' or '\u2028' or '\u2029' || Char.IsUpper(c))); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(2)] - [InlineData(14)] - [InlineData(99)] - [InlineData(-10)] - #endregion - public void CountWhile_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, _) => true)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - #endregion - public void CountWhile_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.CountWhile(index < 0 ? new(-index, true) : (Index)index, (_, _) => true)); - } - - [Theory] - #region Attributes - [InlineData(TestString01)] - [InlineData(TestString02)] - [InlineData(TestString04)] - [InlineData(TestString06)] - #endregion - public void EqualsBuffer(string data) => Assert.Equal(new ReadOnlyStringBuffer(data), new ReadOnlyStringBuffer(data)); - - [Theory] - #region Attributes - [InlineData(TestString01)] - [InlineData(TestString02)] - [InlineData(TestString04)] - [InlineData(TestString06)] - #endregion - public void EqualsMemory(string data) => Assert.True(new ReadOnlyStringBuffer(data).Equals(data.AsMemory())); - - [Theory] - #region Attributes - [InlineData(TestString01)] - [InlineData(TestString02)] - [InlineData(TestString04)] - [InlineData(TestString06)] - #endregion - public void EqualsString(string data) => Assert.True(new ReadOnlyStringBuffer(data).Equals(data)); - - [Theory] - #region Attributes - [InlineData(TestString01)] - [InlineData(TestString02)] - [InlineData(TestString04)] - [InlineData(TestString06)] - #endregion - public void EqualsSpan(string data) => Assert.True(new ReadOnlyStringBuffer(data).Equals(data.AsSpan())); - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 3)] // "Hey" - [InlineData(TestString01, 1, 4)] // "This" - [InlineData(TestString01, 2, 2)] // "Is" - [InlineData(TestString01, 3, 7)] // "A Test " - [InlineData(TestString01, 4, 7)] // " Method" - [InlineData(TestString01, 5, 0)] // "" - [InlineData(TestString01, 6, 1)] // "." - #endregion - #region String ends without newline - [InlineData(TestString02, 0, 3)] // "Hey" - [InlineData(TestString02, 1, 4)] // "This" - [InlineData(TestString02, 2, 2)] // "Is" - [InlineData(TestString02, 3, 7)] // "A Test " - [InlineData(TestString02, 4, 7)] // " Method" - [InlineData(TestString02, 5, 0)] // "" - [InlineData(TestString02, 6, 1)] // "." - #endregion - public void GetLengthOfLine(string data, int lineNumber, int expectedLength) - { - var buffer = new ReadOnlyStringBuffer(data); - buffer.BuildCache(); - Assert.Equal(expectedLength, buffer.GetLengthOfLine(lineNumber)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(20)] - [InlineData(69)] - [InlineData(136)] - [InlineData(-4)] - #endregion - public void GetLengthOfLine_FailsIfEmpty(int lineNumber) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLengthOfLine(lineNumber)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 8)] - [InlineData(TestString01, -1)] - [InlineData(TestString01, -4)] - [InlineData(TestString01, 20)] - [InlineData(TestString01, -20)] - #endregion - public void GetLengthOfLine_FailsIfOutOfRange(string data, int lineNumber) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLengthOfLine(lineNumber)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 3)] // H in "Hey" - [InlineData(TestString01, 3, 3)] // CR in "Hey\r\n" - [InlineData(TestString01, 4, 3)] // LF in "Hey\r\n" - [InlineData(TestString01, 5, 4)] // T in "This" - [InlineData(TestString01, 6, 4)] // h in "This" - [InlineData(TestString01, 13, 7)] // A in "A Test" - [InlineData(TestString01, 15, 7)] // T in "Test" - [InlineData(TestString01, 22, 7)] // M in "Method" - [InlineData(TestString01, 28, 7)] // First CR in "\r\n\r\n." - [InlineData(TestString01, 29, 7)] // First LF in "\r\n\r\n." - [InlineData(TestString01, 30, 0)] // Second CR in "\r\n\r\n." - [InlineData(TestString01, 31, 0)] // Second LF in "\r\n\r\n." - [InlineData(TestString01, 32, 1)] // Dot/point in "\r\n\r\n." - [InlineData(TestString01, 33, 1)] // U2028 in ".\u2028" - [InlineData(TestString01, 34, 0)] // End of file - #endregion - #region String ends without newline - [InlineData(TestString02, 0, 3)] // H in "Hey" - [InlineData(TestString02, 3, 3)] // CR in "Hey\r\n" - [InlineData(TestString02, 4, 3)] // LF in "Hey\r\n" - [InlineData(TestString02, 5, 4)] // T in "This" - [InlineData(TestString02, 6, 4)] // h in "This" - [InlineData(TestString02, 13, 7)] // A in "A Test" - [InlineData(TestString02, 15, 7)] // T in "Test" - [InlineData(TestString02, 22, 7)] // M in "Method" - [InlineData(TestString02, 28, 7)] // First CR in "\r\n\r\n." - [InlineData(TestString02, 29, 7)] // First LF in "\r\n\r\n." - [InlineData(TestString02, 30, 0)] // Second CR in "\r\n\r\n." - [InlineData(TestString02, -2, 0)] // Second LF in "\r\n\r\n." - [InlineData(TestString02, -1, 1)] // Dot/point in "\r\n\r\n." - [InlineData(TestString02, 33, 0)] // End of file - #endregion - public void GetLengthOfLineFromIndex(string data, int index, int expectedLength) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedLength, buffer.GetLengthOfLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(20)] - [InlineData(69)] - [InlineData(136)] - [InlineData(-4)] - #endregion - public void GetLengthOfLineFromIndex_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLengthOfLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - #endregion - public void GetLengthOfLineFromIndex_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLengthOfLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, "Hey")] - [InlineData(TestString01, 1, "This")] - [InlineData(TestString01, 2, "Is")] - [InlineData(TestString01, 3, "A Test ")] - [InlineData(TestString01, 4, " Method")] - [InlineData(TestString01, 5, "")] - [InlineData(TestString01, 6, ".")] - [InlineData(TestString01, 7, "")] // eof - #endregion - #region String ends without newline - [InlineData(TestString02, 0, "Hey")] - [InlineData(TestString02, 1, "This")] - [InlineData(TestString02, 2, "Is")] - [InlineData(TestString02, 3, "A Test ")] - [InlineData(TestString02, 4, " Method")] - [InlineData(TestString02, 5, "")] - [InlineData(TestString02, 6, ".")] - [InlineData(TestString02, 7, "")] // eof - #endregion - #region Single-line strings - [InlineData(TestString05, 0, "!rrrrrrrrr")] - [InlineData(TestString05, 1, "")] // eof - [InlineData(TestString06, 0, "r!!!!!!!!!")] - [InlineData(TestString06, 1, "")] // eof - #endregion - public void GetLine(string data, int lineNumber, string expectedLine) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedLine, buffer.GetLine(lineNumber)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(20)] - [InlineData(69)] - [InlineData(136)] - [InlineData(-4)] - #endregion - public void GetLine_FailsIfEmpty(int lineNumber) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLine(lineNumber)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 8)] - [InlineData(TestString01, -1)] - [InlineData(TestString01, -4)] - [InlineData(TestString01, 20)] - [InlineData(TestString01, -20)] - [InlineData(TestString02, 8)] - [InlineData(TestString05, 2)] - #endregion - public void GetLine_FailsIfOutOfRange(string data, int lineNumber) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLine(lineNumber)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, "Hey")] // H in "Hey" - [InlineData(TestString01, 2, "Hey")] // y in "Hey\r\n" - [InlineData(TestString01, 3, "Hey")] // CR in "Hey\r\n" - [InlineData(TestString01, 4, "Hey")] // LF in "Hey\r\n" - [InlineData(TestString01, 5, "This")] // T in "This" - [InlineData(TestString01, 6, "This")] // h in "This" - [InlineData(TestString01, 11, "Is")] // s in "Is" - [InlineData(TestString01, 12, "Is")] // U2029 before "A Test" - [InlineData(TestString01, 13, "A Test ")] // A in "A Test" - [InlineData(TestString01, 15, "A Test ")] // T in "Test" - [InlineData(TestString01, 22, " Method")] // M in "Method" - [InlineData(TestString01, 28, " Method")] // First CR in "\r\n\r\n." - [InlineData(TestString01, 29, " Method")] // First LF in "\r\n\r\n." - [InlineData(TestString01, 30, "")] // Second CR in "\r\n\r\n." - [InlineData(TestString01, 31, "")] // Second LF in "\r\n\r\n." - [InlineData(TestString01, 32, ".")] // Dot/point in "\r\n\r\n." - [InlineData(TestString01, 33, ".")] // U2028 in ".\u2028" - [InlineData(TestString01, 34, "")] // End of file - #endregion - #region String ends without newline - [InlineData(TestString02, 0, "Hey")] // H in "Hey" - [InlineData(TestString02, 2, "Hey")] // y in "Hey\r\n" - [InlineData(TestString02, 3, "Hey")] // CR in "Hey\r\n" - [InlineData(TestString02, 4, "Hey")] // LF in "Hey\r\n" - [InlineData(TestString02, 5, "This")] // T in "This" - [InlineData(TestString02, 6, "This")] // h in "This" - [InlineData(TestString02, 11, "Is")] // s in "Is" - [InlineData(TestString02, 12, "Is")] // U2029 before "A Test" - [InlineData(TestString02, 13, "A Test ")] // A in "A Test" - [InlineData(TestString02, 15, "A Test ")] // T in "Test" - [InlineData(TestString02, 22, " Method")] // M in "Method" - [InlineData(TestString02, 28, " Method")] // First CR in "\r\n\r\n." - [InlineData(TestString02, 29, " Method")] // First LF in "\r\n\r\n." - [InlineData(TestString02, 30, "")] // Second CR in "\r\n\r\n." - [InlineData(TestString02, -2, "")] // Second LF in "\r\n\r\n." - [InlineData(TestString02, -1, ".")] // Dot/point in "\r\n\r\n." - [InlineData(TestString02, 33, "")] // End of file - #endregion - public void GetLineFromIndex(string data, int index, string expectedLine) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedLine, buffer.GetLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - #endregion - public void GetLineFromIndex_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(20)] - [InlineData(69)] - [InlineData(136)] - [InlineData(-4)] - #endregion - public void GetLineFromIndex_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLineFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 0)] // H in "Hey" - [InlineData(TestString01, 3, 0)] // CR in "Hey\r\n" - [InlineData(TestString01, 4, 0)] // LF in "Hey\r\n" - [InlineData(TestString01, 5, 1)] // T in "This" - [InlineData(TestString01, 6, 1)] // h in "This" - [InlineData(TestString01, 13, 3)] // A in "A Test" - [InlineData(TestString01, 15, 3)] // T in "Test" - [InlineData(TestString01, 22, 4)] // M in "Method" - [InlineData(TestString01, 28, 4)] // First CR in "\r\n\r\n." - [InlineData(TestString01, 29, 4)] // First LF in "\r\n\r\n." - [InlineData(TestString01, 30, 5)] // Second CR in "\r\n\r\n." - [InlineData(TestString01, 31, 5)] // Second LF in "\r\n\r\n." - [InlineData(TestString01, 32, 6)] // Dot/point in "\r\n\r\n." - [InlineData(TestString01, 33, 6)] // U2028 in ".\u2028" - [InlineData(TestString01, 34, 7)] // End of file - #endregion - #region String ends without newline - [InlineData(TestString02, 0, 0)] // H in "Hey" - [InlineData(TestString02, 3, 0)] // CR in "Hey\r\n" - [InlineData(TestString02, 4, 0)] // LF in "Hey\r\n" - [InlineData(TestString02, 5, 1)] // T in "This" - [InlineData(TestString02, 6, 1)] // h in "This" - [InlineData(TestString02, 13, 3)] // A in "A Test" - [InlineData(TestString02, 15, 3)] // T in "Test" - [InlineData(TestString02, 22, 4)] // M in "Method" - [InlineData(TestString02, 28, 4)] // First CR in "\r\n\r\n." - [InlineData(TestString02, 29, 4)] // First LF in "\r\n\r\n." - [InlineData(TestString02, 30, 5)] // Second CR in "\r\n\r\n." - [InlineData(TestString02, -2, 5)] // Second LF in "\r\n\r\n." - [InlineData(TestString02, -1, 6)] // Dot/point in "\r\n\r\n." - [InlineData(TestString02, 33, 7)] // EOF - #endregion - public void GetLineNumberFromIndex(string data, int index, int expectedLineNumber) - { - var buffer = new ReadOnlyStringBuffer(data); - buffer.BuildCache(); - Assert.Equal(expectedLineNumber, buffer.GetLineNumberFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - #endregion - public void GetLineNumberFromIndex_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLineNumberFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(20)] - [InlineData(69)] - [InlineData(136)] - [InlineData(-4)] - #endregion - public void GetLineNumberFromIndex_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLineNumberFromIndex(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 0, 0)] // H in "Hey" - [InlineData(TestString01, 3, 0, 3)] // CR in "Hey\r\n" - [InlineData(TestString01, 4, 0, 4)] // LF in "Hey\r\n" - [InlineData(TestString01, 5, 1, 0)] // T in "This" - [InlineData(TestString01, 6, 1, 1)] // h in "This" - [InlineData(TestString01, 13, 3, 0)] // A in "A Test" - [InlineData(TestString01, 15, 3, 2)] // T in "Test" - [InlineData(TestString01, 22, 4, 1)] // M in "Method" - [InlineData(TestString01, 28, 4, 7)] // First CR in "\r\n\r\n." - [InlineData(TestString01, -5, 4, 8)] // First LF in "\r\n\r\n." - [InlineData(TestString01, 30, 5, 0)] // Second CR in "\r\n\r\n." - [InlineData(TestString01, -3, 5, 1)] // Second LF in "\r\n\r\n." - [InlineData(TestString01, 32, 6, 0)] // Dot/point in "\r\n\r\n." - [InlineData(TestString01, 33, 6, 1)] // U2028 in ".\u2028" - #endregion - #region String ends without newline - [InlineData(TestString02, 0, 0, 0)] // H in "Hey" - [InlineData(TestString02, 3, 0, 3)] // CR in "Hey\r\n" - [InlineData(TestString02, 4, 0, 4)] // LF in "Hey\r\n" - [InlineData(TestString02, 5, 1, 0)] // T in "This" - [InlineData(TestString02, 6, 1, 1)] // h in "This" - [InlineData(TestString02, 13, 3, 0)] // A in "A Test" - [InlineData(TestString02, 15, 3, 2)] // T in "Test" - [InlineData(TestString02, 22, 4, 1)] // M in "Method" - [InlineData(TestString02, 28, 4, 7)] // First CR in "\r\n\r\n." - [InlineData(TestString02, 29, 4, 8)] // First LF in "\r\n\r\n." - [InlineData(TestString02, 30, 5, 0)] // Second CR in "\r\n\r\n." - [InlineData(TestString02, -2, 5, 1)] // Second LF in "\r\n\r\n." - [InlineData(TestString02, -1, 6, 0)] // Dot/point in "\r\n\r\n." - #endregion - public void GetLocationDetails(string data, int index, int expectedLine, int expectedColumn) - { - var buffer = new ReadOnlyStringBuffer(data); - var location = buffer.GetLocationDetails(index < 0 ? new(-index, true) : (Index)index); - - Assert.Equal(expectedLine, location.Line); - Assert.Equal(expectedColumn, location.Column); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 99)] - [InlineData(TestString01, 34)] // EOF is not accepted for this method - [InlineData(TestString01, 35)] - [InlineData(TestString01, -35)] - #endregion - public void GetLocationDetails_FailsIfOutOfRange(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Throws(() => buffer.GetLocationDetails(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region Attributes - [InlineData(0)] - [InlineData(20)] - [InlineData(69)] - [InlineData(136)] - [InlineData(-4)] - #endregion - public void GetLocationDetails_FailsIfEmpty(int index) - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.Throws(() => buffer.GetLocationDetails(index < 0 ? new(-index, true) : (Index)index)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0)] - [InlineData(TestString01, 3)] - [InlineData(TestString01, 7)] - [InlineData(TestString01, 8)] - [InlineData(TestString01, 10)] - [InlineData(TestString01, 12)] - [InlineData(TestString01, 15)] - [InlineData(TestString01, 21)] - [InlineData(TestString01, 24)] - [InlineData(TestString01, 26)] - [InlineData(TestString01, 27)] - [InlineData(TestString01, 30)] - [InlineData(TestString01, 32)] - [InlineData(TestString01, 33)] - #endregion - #region String ends with newline - [InlineData(TestString02, 0)] - [InlineData(TestString02, 2)] - [InlineData(TestString02, 5)] - [InlineData(TestString02, 9)] - [InlineData(TestString02, 10)] - [InlineData(TestString02, 12)] - [InlineData(TestString02, 15)] - [InlineData(TestString02, 19)] - [InlineData(TestString02, 23)] - [InlineData(TestString02, 27)] - [InlineData(TestString02, 29)] - [InlineData(TestString02, 31)] - #endregion - #region String with a lot of whitespace - [InlineData(TestString03, 0)] - [InlineData(TestString03, 2)] - [InlineData(TestString03, 15)] - [InlineData(TestString03, 29)] - [InlineData(TestString03, 30)] - [InlineData(TestString03, 42)] - [InlineData(TestString03, 55)] - [InlineData(TestString03, 69)] - [InlineData(TestString03, 73)] - #endregion - #region String with a single line - [InlineData(TestString05, 0)] - [InlineData(TestString05, 2)] - [InlineData(TestString05, 5)] - [InlineData(TestString05, 9)] - [InlineData(TestString06, 1)] - [InlineData(TestString06, 4)] - [InlineData(TestString06, 6)] - [InlineData(TestString06, 8)] - #endregion - public void IndexAccessor(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(data[index], buffer[index]); - Assert.Equal(data[index], buffer[index < 0 ? new Index(-index, true) : (Index)index]); - } - - [Theory] - #region Attributes - [InlineData(TestString01, 7)] - [InlineData(TestString02, 7)] - [InlineData(TestString03, 4)] - [InlineData(TestString04, 3)] - [InlineData(TestString05, 1)] - [InlineData(TestString06, 1)] - [InlineData(TestString07, 3)] - #endregion - public void LineCount_MatchesActualLineCountWithNoEof(string data, int expectedLineCount) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.Equal(expectedLineCount, buffer.LineCount); - } - - [Theory] - #region Attributes - [InlineData(TestString02)] - [InlineData(TestString03)] - [InlineData(TestString04)] - [InlineData(TestString06)] - #endregion - public void Constructor_CharArray(string data) - { - var array = data.ToCharArray(); - Assert.Equal(data, array); - Assert.True(new ReadOnlyStringBuffer(array).Equals(data)); - } - - [Theory] - #region Attributes - [InlineData(TestString01)] - [InlineData(TestString02)] - [InlineData(TestString03)] - [InlineData(TestString04)] - [InlineData(TestString05)] - [InlineData(TestString06)] - #endregion - public unsafe void Constructor_BytePointer(string data) - { - var byteCount = Encoding.Default.GetByteCount(data); - - fixed (byte* pointer = Encoding.Default.GetBytes(data)) - { - Assert.Equal(data, Encoding.Default.GetString(pointer, byteCount)); - Assert.True(new ReadOnlyStringBuffer(pointer, byteCount).Equals(data)); - } - } - - [Theory] - #region Attributes - [InlineData(TestString02)] - [InlineData(TestString03)] - [InlineData(TestString04)] - [InlineData(TestString06)] - #endregion - public void Constructor_ByteArray(string data) - { - var array = Encoding.Default.GetBytes(data); - Assert.True(new ReadOnlyStringBuffer(array).Equals(data)); - } - - [Theory] - #region Attributes - [InlineData(TestString02)] - [InlineData(TestString03)] - [InlineData(TestString04)] - [InlineData(TestString06)] - #endregion - public void Constructor_ReadOnlySpan(string data) - { - ReadOnlySpan span = data.AsSpan(); - Assert.True(new ReadOnlyStringBuffer(span).Equals(data)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 4, "Hey\r")] - [InlineData(TestString01, 3, 2, "\r\n")] - [InlineData(TestString01, 5, 1, "T")] - [InlineData(TestString01, 7, 6, "is\rIs\u2029")] - [InlineData(TestString01, 12, 8, "\u2029A Test ")] - [InlineData(TestString01, 16, 4, "est ")] - [InlineData(TestString01, 30, 3, "\r\n.")] - [InlineData(TestString01, -3, 3, "\n.\u2028")] - #endregion - public void Slice_CharSpan(string data, int index, int length, string expectedSlice) - { - var buffer = new ReadOnlyStringBuffer(data); - var slice = buffer.Slice(index < 0 ? new(-index, true) : (Index)index, length); - - Assert.Equal(expectedSlice, new string(slice)); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, "Hey\r")] - [InlineData(TestString01, 3, "\r\n")] - [InlineData(TestString01, 5, "T")] - [InlineData(TestString01, 7, "is\rIs\u2029")] - [InlineData(TestString01, 12, "\u2029A Test ")] - [InlineData(TestString01, 16, "est ")] - [InlineData(TestString01, 30, "\r\n.")] - [InlineData(TestString01, -3, "\n.\u2028")] - #endregion - public void TrySlice_CharSpan(string data, int index, string expectedSlice) - { - Span span = stackalloc char[expectedSlice.Length]; - - var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TrySlice(index < 0 ? new(-index, true) : (Index)index, span)); - - Assert.Equal(expectedSlice, span); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, 4, "Hey\r")] - [InlineData(TestString01, 3, 5, "\r\n")] - [InlineData(TestString01, 5, 6, "T")] - [InlineData(TestString01, 7, 13, "is\rIs\u2029")] - [InlineData(TestString01, 12, 20, "\u2029A Test ")] - [InlineData(TestString01, 16, 20, "est ")] - [InlineData(TestString01, 30, 33, "\r\n.")] - [InlineData(TestString01, 31, 34, "\n.\u2028")] - #endregion - public void Slice_Range(string data, int startIndex, int endIndex, string expectedSlice) - { - Span span = stackalloc char[expectedSlice.Length]; - - var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TrySlice(startIndex..endIndex, span)); - - Assert.Equal(expectedSlice, span); - } - - [Fact] - public void Slice_Range_FailsIfSpanTooSmall() - { - var buffer = new ReadOnlyStringBuffer(TestString05); - Assert.False(buffer.TrySlice(0..7, stackalloc char[3])); - } - - [Fact] - public void ToString_SameAsInputData() - { - var buffer = new ReadOnlyStringBuffer(TestString02); - Assert.Equal(TestString02, buffer.ToString()); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0)] - [InlineData(TestString01, 3)] - [InlineData(TestString01, 7)] - [InlineData(TestString01, 8)] - [InlineData(TestString01, 10)] - [InlineData(TestString01, 12)] - [InlineData(TestString01, 15)] - [InlineData(TestString01, 21)] - [InlineData(TestString01, 24)] - [InlineData(TestString01, 26)] - [InlineData(TestString01, 27)] - [InlineData(TestString01, 30)] - [InlineData(TestString01, 32)] - [InlineData(TestString01, 33)] - #endregion - #region String ends with newline - [InlineData(TestString02, 0)] - [InlineData(TestString02, 2)] - [InlineData(TestString02, 5)] - [InlineData(TestString02, 9)] - [InlineData(TestString02, 10)] - [InlineData(TestString02, 12)] - [InlineData(TestString02, 15)] - [InlineData(TestString02, 19)] - [InlineData(TestString02, 23)] - [InlineData(TestString02, 27)] - [InlineData(TestString02, 29)] - [InlineData(TestString02, 31)] - #endregion - #region String with a lot of whitespace - [InlineData(TestString03, 0)] - [InlineData(TestString03, 2)] - [InlineData(TestString03, 15)] - [InlineData(TestString03, 29)] - [InlineData(TestString03, 30)] - [InlineData(TestString03, 42)] - [InlineData(TestString03, 55)] - [InlineData(TestString03, 69)] - [InlineData(TestString03, 73)] - #endregion - #region String with a single line - [InlineData(TestString05, 0)] - [InlineData(TestString05, 2)] - [InlineData(TestString05, 5)] - [InlineData(TestString05, 9)] - [InlineData(TestString06, 1)] - [InlineData(TestString06, 4)] - [InlineData(TestString06, 6)] - [InlineData(TestString06, 8)] - #endregion - public void TryGetChar(string data, int index) - { - var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TryGetChar(index < 0 ? new(-index, true) : (Index)index, out char item)); - Assert.Equal(data[index], item); - } - - [Fact] - public void TryGetChar_FollowsSpecificEofConvention() - { - var buffer = new ReadOnlyStringBuffer(TestString05); - Assert.False(buffer.TryGetChar(10, out char item)); - Assert.Equal('\0', item); - } - - [Fact] - public void TryGetChar_FailsIfEmpty() - { - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.False(buffer.TryGetChar(0, out char item)); - Assert.Equal('\0', item); - } - - [Fact] - public void TryGetChar_FailsIfOutOfRange() - { - var buffer = new ReadOnlyStringBuffer(TestString06); - Assert.False(buffer.TryGetChar(14, out char item)); - Assert.Equal('\0', item); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, "Hey")] - [InlineData(TestString01, 1, "This")] - [InlineData(TestString01, 2, "Is")] - [InlineData(TestString01, 3, "A Test ")] - [InlineData(TestString01, 4, " Method")] - [InlineData(TestString01, 5, "")] - [InlineData(TestString01, 6, ".")] - [InlineData(TestString01, 7, "")] // eof - #endregion - #region String ends without newline - [InlineData(TestString02, 0, "Hey")] - [InlineData(TestString02, 1, "This")] - [InlineData(TestString02, 2, "Is")] - [InlineData(TestString02, 3, "A Test ")] - [InlineData(TestString02, 4, " Method")] - [InlineData(TestString02, 5, "")] - [InlineData(TestString02, 6, ".")] - [InlineData(TestString02, 7, "")] // eof - #endregion - #region Single-line strings - [InlineData(TestString05, 0, "!rrrrrrrrr")] - [InlineData(TestString05, 1, "")] // eof - [InlineData(TestString06, 0, "r!!!!!!!!!")] - [InlineData(TestString06, 1, "")] // eof - #endregion - public void TryGetLine(string data, int lineNumber, string expectedLine) - { - Span span = stackalloc char[expectedLine.Length]; - - var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TryGetLine(lineNumber, span)); - Assert.Equal(expectedLine, span); - } - - [Fact] - public void TryGetLine_FailsIfEmpty() - { - Span span = []; - - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.False(buffer.TryGetLine(0, span)); - Assert.True(span.IsEmpty); - } - - [Fact] - public void TryGetLine_FailsIfOutOfRange() - { - Span span = stackalloc char[10]; - - var buffer = new ReadOnlyStringBuffer(TestString05); - const string emptySequence = "\0\0\0\0\0\0\0\0\0\0"; - - Assert.False(buffer.TryGetLine(2, span)); - Assert.Equal(emptySequence, span); - - Assert.False(buffer.TryGetLine(-1, span)); - Assert.Equal(emptySequence, span); - } - - [Fact] - public void TryGetLine_FalseIfSpanTooSmall() - { - Span span = stackalloc char[5]; - - var buffer = new ReadOnlyStringBuffer(TestString05); - Assert.False(buffer.TryGetLine(0, span)); - Assert.Equal("\0\0\0\0\0", span); - } - - [Theory] - #region String ends with newline - [InlineData(TestString01, 0, "Hey")] // H in "Hey" - [InlineData(TestString01, 3, "Hey")] // CR in "Hey\r\n" - [InlineData(TestString01, 4, "Hey")] // LF in "Hey\r\n" - [InlineData(TestString01, 5, "This")] // T in "This" - [InlineData(TestString01, 13, "A Test ")] // A in "A Test" - [InlineData(TestString01, 15, "A Test ")] // T in "Test" - [InlineData(TestString01, 22, " Method")] // M in "Method" - [InlineData(TestString01, 28, " Method")] // First CR in "\r\n\r\n." - [InlineData(TestString01, 29, " Method")] // First LF in "\r\n\r\n." - [InlineData(TestString01, 30, "")] // Second CR in "\r\n\r\n." - [InlineData(TestString01, 31, "")] // Second LF in "\r\n\r\n." - [InlineData(TestString01, 32, ".")] // Dot/point in Second LF in "\r\n\r\n." - [InlineData(TestString01, 33, ".")] // U2028 in ".\u2028" - [InlineData(TestString01, 34, "")] // End of file - #endregion - #region String ends without newline - [InlineData(TestString02, 0, "Hey")] // H in "Hey" - [InlineData(TestString02, 3, "Hey")] // CR in "Hey\r\n" - [InlineData(TestString02, 4, "Hey")] // LF in "Hey\r\n" - [InlineData(TestString02, 5, "This")] // T in "This" - [InlineData(TestString02, 13, "A Test ")] // A in "A Test" - [InlineData(TestString02, 15, "A Test ")] // T in "Test" - [InlineData(TestString02, 22, " Method")] // M in "Method" - [InlineData(TestString02, 28, " Method")] // First CR in "\r\n\r\n." - [InlineData(TestString02, 29, " Method")] // First LF in "\r\n\r\n." - [InlineData(TestString02, 30, "")] // Second CR in "\r\n\r\n." - [InlineData(TestString02, -2, "")] // Second LF in "\r\n\r\n." - [InlineData(TestString02, -1, ".")] // Dot/point in Second LF in "\r\n\r\n." - [InlineData(TestString02, 33, "")] // End of file - #endregion - public void TryGetLineFromIndex(string data, int index, string expectedLine) - { - Span span = stackalloc char[expectedLine.Length]; - - var buffer = new ReadOnlyStringBuffer(data); - Assert.True(buffer.TryGetLineFromIndex(index < 0 ? new(-index, true) : (Index)index, span)); - Assert.Equal(expectedLine, span.ToString()); - } - - [Fact] - public void TryGetLineFromIndex_FailsIfEmpty() - { - Span span = []; - - var buffer = new ReadOnlyStringBuffer(String.Empty); - Assert.False(buffer.TryGetLineFromIndex(0, span)); - Assert.True(span.IsEmpty); - } - - [Fact] - public void TryGetLineFromIndex_FailsIfOutOfRange() - { - Span span = stackalloc char[10]; - - var buffer = new ReadOnlyStringBuffer(TestString05); - const string emptySequence = "\0\0\0\0\0\0\0\0\0\0"; - - Assert.False(buffer.TryGetLineFromIndex(12, span)); - Assert.Equal(emptySequence, span); - - Assert.False(buffer.TryGetLineFromIndex(^12, span)); - Assert.Equal(emptySequence, span); - } - - [Fact] - public void TryGetLineFromIndex_FalseIfSpanTooSmall() - { - Span span = stackalloc char[5]; - - var buffer = new ReadOnlyStringBuffer(TestString05); - Assert.False(buffer.TryGetLineFromIndex(0, span)); - Assert.Equal("\0\0\0\0\0", span); - } -} From c146f26633726c277ca6d80b4e7985c7183b9ea6 Mon Sep 17 00:00:00 2001 From: Matthew Date: Mon, 10 Aug 2026 23:44:13 +0100 Subject: [PATCH 08/11] perf(lexer)!: Refactor the lexer for performance (using UTF-8 and ASCII extensions instead of UTF-16 string methods) [1/?] Code is fundamentally broken, so CI will fail. Lots of TODOs in code. Essentially, this is very bad code LMFAO (for now). Signed-off-by: Matthew Refs: #49 --- src/RSML.Language.Lexing/ILexer.cs | 20 +- src/RSML.Language.Lexing/Tokens/Token.cs | 10 +- src/RSML.Language.Lexing/Utf8Lexer.cs | 289 ++++++++++++++++++ src/RSML.Language.Parsing/IParser.cs | 2 +- src/RSML.Language.Parsing/Parser.cs | 2 + .../Interpreter.cs | 4 +- 6 files changed, 309 insertions(+), 18 deletions(-) create mode 100644 src/RSML.Language.Lexing/Utf8Lexer.cs diff --git a/src/RSML.Language.Lexing/ILexer.cs b/src/RSML.Language.Lexing/ILexer.cs index 4c551ff..d33e10c 100644 --- a/src/RSML.Language.Lexing/ILexer.cs +++ b/src/RSML.Language.Lexing/ILexer.cs @@ -1,8 +1,10 @@ +using System; +using System.Buffers; using System.Collections.Generic; -using OceanApocalypse.RSML.Language.Lexing.Tokens; -using OceanApocalypse.RSML.Abstractions; using OceanApocalypse.RSML.Abstractions.Diagnostics; +using OceanApocalypse.RSML.Abstractions.Toolchain; +using OceanApocalypse.RSML.Language.Lexing.Tokens; namespace OceanApocalypse.RSML.Language.Lexing; @@ -10,24 +12,18 @@ namespace OceanApocalypse.RSML.Language.Lexing; /// /// Represents a lexer for RSML. /// -/// -/// :::tip[Avoid starting from scratch] -/// If you want to add content on top of a lexer, without overriding -/// the extra functionality it adds, you might want to take a look at -/// . -/// ::: -/// -public interface ILexer : IToolchainComponent +public interface ILexer : IToolchainComponent + where TInput : unmanaged, IEquatable { /// /// Tokenizes a source passed to the lexer. /// /// The tokens. - IEnumerable Lex(); + IEnumerable Lex(ReadOnlySequence data); /// /// Returns the next token. /// /// The next token. - Result GetNextToken(); + Result GetNextToken(ref SequenceReader reader); } diff --git a/src/RSML.Language.Lexing/Tokens/Token.cs b/src/RSML.Language.Lexing/Tokens/Token.cs index 4235ccb..b58b1a6 100644 --- a/src/RSML.Language.Lexing/Tokens/Token.cs +++ b/src/RSML.Language.Lexing/Tokens/Token.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace OceanApocalypse.RSML.Language.Lexing.Tokens; @@ -8,14 +9,15 @@ namespace OceanApocalypse.RSML.Language.Lexing.Tokens; /// Represents a RSML token. /// /// An integer that identifies the type of token. -/// The token's value. -/// The range where the token occurs. -public record struct Token(TokenKind Kind, object? Value, Range Range) +/// The offset at which the token begins. +/// The token's length. +[StructLayout(LayoutKind.Sequential)] +public record struct Token(TokenKind Kind, long StartOffset, long Length) { /// /// Empty token. Used when something goes wrong. /// - public readonly static Token Empty = new(TokenKind.Unknown, null, new()); + public readonly static Token Empty = new(TokenKind.Unknown, 0L, 0L); /// /// Gets the token kind that applies to the given keyword or keyword modifier. diff --git a/src/RSML.Language.Lexing/Utf8Lexer.cs b/src/RSML.Language.Lexing/Utf8Lexer.cs new file mode 100644 index 0000000..c250c71 --- /dev/null +++ b/src/RSML.Language.Lexing/Utf8Lexer.cs @@ -0,0 +1,289 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Data; + +using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Diagnostics; +using OceanApocalypse.RSML.Abstractions.Panic; +using OceanApocalypse.RSML.Abstractions.Toolchain; +using OceanApocalypse.RSML.Language.Lexing.Diagnostics; +using OceanApocalypse.RSML.Language.Lexing.Tokens; + +namespace OceanApocalypse.RSML.Language.Lexing; + +/// +/// An implementation of a RSML lexer backed by a given UTF-8 buffer. +/// +/// +/// Initializes a new lexer with a given configuration and diagnostic collector. +/// +/// A collector with all the diagnostics that were and will be emitted. +/// Configurations for the toolchain components. +public class Utf8Lexer(DiagnosticCollector diagnosticCollector, ToolchainConfiguration? configuration = null) : ILexer +{ + private bool isDisposed; + private bool wasUsed; + + /// + /// A collector containing all emitted diagnostics. + /// + protected DiagnosticCollector Diagnostics { get; } = diagnosticCollector; + + /// + public ToolchainConfiguration Configuration { get; protected set; } = configuration ?? ToolchainConfiguration.Default; + + /// + /// :::note[Diagnostic output] + /// This method does not add diagnostics to the collector + /// (): it only returns them when it + /// proves necessary. + /// ::: + /// + /// + public Result GetNextToken(ref SequenceReader reader) + { + wasUsed = true; + SkipWhitespaceAndComments(ref reader); + long startLoc = reader.Consumed; + + if (reader.End || !reader.TryPeek(out byte b)) + return Result.Success(new Token(TokenKind.Eof, startLoc, 0L)); + + char c = (char)b; + + // strings + if (c == '"') + return ScanStringLiteral(ref reader, startLoc); + + // number literals + if (b.IsAsciiDigit()) + return ScanNumber(ref reader, b, startLoc); + + // identifiers and keywords + if (b.IsAsciiLetter() || c == '_') + return ScanIdentifierOrKeyword(ref reader, startLoc); + + // standard library identifiers + if (c == '$') + return ScanStdIdentifier(ref reader, startLoc); + + // member access notation + if (c == '.') + return Result.Success(new Token(TokenKind.MemberAccess, startLoc, 1L)); + + // punctuation + if (b.IsRsmlPunctuation()) + return ScanPunctuation(ref reader, startLoc); + + // todo: check for comments if Configuration.EmitComments is enabled + + return Result.Failure(new( + LexerErrorCodes.FailedToLexToken, + "Tried all possible token logic paths, but none was true. This likely means you used a character not recognized by the lexer," + + "but it may also mean the lexer is mal-functioning.", + Severity.Critical + )); + } + + /// + public IEnumerable Lex(ReadOnlySequence data) + { + wasUsed = true; + int failedRuns = 0; + + while (Configuration.MaximumAllowedFailuresPerComponent <= 0 || failedRuns < Configuration.MaximumAllowedFailuresPerComponent) + { + // todo: create reader here + var token = GetNextToken(ref reader); + + if (token.IsError) + { + Diagnostics.Add(token.Error); + failedRuns++; + continue; + } + + if (token.Value.Kind == TokenKind.Eof) + yield break; + + else + yield return token.Value; + } + + throw new ExceededMaxAmountOfFailuresException( + $"This instance of the lexer was allowed to fail up to {Configuration.MaximumAllowedFailuresPerComponent} times, yet it failed {failedRuns}." + ); + } + + private static Result ScanNumber(ref SequenceReader reader, byte startChar, long startLoc) + { + const byte underscore = (byte)'_'; + const byte dot = (byte)'.'; + + byte b = startChar; + bool hasDotSeparator = false; + + do + { + reader.Advance(1); + + if (b == dot) + hasDotSeparator = true; + + } while (!reader.End && reader.TryPeek(out b) && (b.IsAsciiDigit() || b == underscore || (b == dot && !hasDotSeparator))); + + return Result.Success(new Token(TokenKind.NumericLiteral, startLoc, reader.Consumed - startLoc)); + } + + // todo: fix the method below + private Result ScanStringLiteral(ref SequenceReader reader, long startLoc) + { + cursor++; + bool escaping = false; + + while (cursor < Sequence.Length) + { + if (Sequence[cursor].IsNewline()) + { + return Result.Failure(new( + LexerErrorCodes.UnterminatedStringLiteral, + Sequence.GetLocationDetails((Index)startLoc), + Sequence.GetLocationDetails((Index)cursor), + "A string literal must begin and end in the same line.", + Severity.Error + )); + } + + if (Sequence[cursor] == '"' && !escaping) + break; + + if (Sequence[cursor] == '\\') + escaping = !escaping; + + cursor++; + } + + if (cursor < Sequence.Length) + cursor++; // skip end quote if anything beyond it + + return Result.Success(new Token(TokenKind.StringLiteral, null, startLoc..cursor)); + } + + // todo: fix the method below + private Result ScanStdIdentifier(ref SequenceReader reader, long startLoc) + { + // this points to h in $helloWorld broski + int afterStdSymbolIndex = ++cursor; // we also skip past it to avoid extra checks in while loop + + while (cursor < Sequence.Length && (Char.IsAsciiLetterOrDigit(Sequence[cursor]) || Sequence[cursor] == '_')) + cursor++; + + return cursor == afterStdSymbolIndex + ? Result.Failure(new( + LexerErrorCodes.ExpectedStdIdentifier, + Sequence.GetLocationDetails((Index)startLoc), + Sequence.GetLocationDetails((Index)cursor), + "Expected a standard library identifier, yet there was no valid identifier after the $ symbol.", + Severity.Error + )) + : Result.Success(new Token(TokenKind.StandardLibraryIdentifier, null, startLoc..cursor)); + } + + // todo: fix the method below + private Result ScanIdentifierOrKeyword(ref SequenceReader reader, long startLoc) + { + while (cursor < Sequence.Length && (Char.IsAsciiLetterOrDigit(Sequence[cursor]) || Sequence[cursor] == '_')) + cursor++; + + Range range = startLoc..cursor; + + if (Keywords.Contains(Sequence[range])) + { + var token = new Token(Token.GetKeywordKind(Sequence[range]), null, range); // is keyword + + return token.Kind == TokenKind.Unknown + ? Result.Failure(new( + LexerErrorCodes.FailedToIdentifyKeyword, + Sequence.GetLocationDetails(startLoc), + Sequence.GetLocationDetails(cursor), + "Despite identifying the object in question as a keyword, the lexer failed to resolve exactly which keyword it was." + + "This likely means the keyword in question is reserved for future use, but isn't implemented yet.", + Severity.Error + )) + : Result.Success(token); + } + else + { + return Result.Success(new Token(TokenKind.Identifier, null, range)); // is identifier + } + } + + // todo: fix the method below + private Result ScanPunctuation(ref SequenceReader reader, long startLoc) + { + char c = Sequence[cursor]; + char? peeked = cursor + 1 >= Sequence.Length ? null : Sequence[++cursor]; // dont error out if out of bounds + TokenKind kind = Token.GetPunctuationKind(c, peeked); + + return kind == TokenKind.Unknown + ? Result.Failure(new( + LexerErrorCodes.FailedToIdentifyPunctuation, + Sequence.GetLocationDetails(startLoc), + Sequence.GetLocationDetails(peeked is null ? cursor - 1 : cursor), + "Despite identifying the object in question as punctuation, the lexer failed to resolve exactly which punctuation it was." + + "This might mean the punctuation in question is reserved for future use, and not implemented yet.", + Severity.Error + )) + : Result.Success(new Token(kind, null, startLoc..cursor)); + } + + private void SkipWhitespaceAndComments(ref SequenceReader reader) + { + while (!reader.End && reader.TryPeek(out byte b)) + { + if (b.IsAsciiWhitespace()) + reader.Advance(1); + + else if (b == (byte)'#' && !Configuration.EmitComments) + reader.TryAdvanceToAny([(byte)'\r', (byte)'\n'], advancePastDelimiter: true); + + else + break; + } + } + + /// + public void Inject(ToolchainConfiguration configuration) + { + if (wasUsed) + throw new ReadOnlyException("The configuration has already been apply and cannot be altered."); + + Configuration = configuration; + } + + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Disposes of internally used unmanaged resources. + /// + /// If true, also disposes of managed resources. + protected virtual void Dispose(bool disposing) + { + if (isDisposed) + return; + + // unmanaged things here + + if (disposing) + { } // managed things + + isDisposed = true; + } +} diff --git a/src/RSML.Language.Parsing/IParser.cs b/src/RSML.Language.Parsing/IParser.cs index b5abc56..aab6002 100644 --- a/src/RSML.Language.Parsing/IParser.cs +++ b/src/RSML.Language.Parsing/IParser.cs @@ -1,4 +1,4 @@ -using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Toolchain; namespace OceanApocalypse.RSML.Language.Parsing; diff --git a/src/RSML.Language.Parsing/Parser.cs b/src/RSML.Language.Parsing/Parser.cs index e0803e3..77dfe04 100644 --- a/src/RSML.Language.Parsing/Parser.cs +++ b/src/RSML.Language.Parsing/Parser.cs @@ -9,6 +9,8 @@ namespace OceanApocalypse.RSML.Language.Parsing; /// public abstract class Parser : IParser { + // todo: make this implement IParser correctly + private bool isDisposed; /// diff --git a/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs b/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs index f0f1685..183d69e 100644 --- a/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs +++ b/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs @@ -1,6 +1,7 @@ using System; using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Toolchain; namespace OceanApocalypse.RSML.Toolchain.Extensibility.Execution; @@ -9,7 +10,8 @@ namespace OceanApocalypse.RSML.Toolchain.Extensibility.Execution; /// public abstract class Interpreter : IToolchainComponent { - // todo: add necessary content to IInterpreter + // todo: make this implement IToolchainComponent correctly + private bool isDisposed; /// From 71a456e342470cc0d3ae024c15cf178c3eb6875a Mon Sep 17 00:00:00 2001 From: Matthew Date: Tue, 11 Aug 2026 22:24:29 +0100 Subject: [PATCH 09/11] perf(lexer)!: Refactor the lexer for performance (using UTF-8 and ASCII extensions instead of UTF-16 string methods) [2/3] Code is fundamentally broken, so CI will fail. Lots of TODOs in code. Essentially, this is very bad code LMFAO (for now). Signed-off-by: Matthew Refs: #49 --- props/Defaults.props | 2 +- .../AbsolutePosition.cs | 169 ++++++++++++++++++ .../Diagnostic.cs | 71 ++++---- src/RSML.Abstractions/Extensions.cs | 7 + .../Diagnostics/LexerErrorCodes.cs | 1 + src/RSML.Language.Lexing/ILexer.cs | 23 +-- src/RSML.Language.Lexing/Utf8/IUnsafeLexer.cs | 19 ++ src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs | 42 +++++ .../{ => Utf8}/Utf8Lexer.cs | 108 ++++++++--- 9 files changed, 371 insertions(+), 71 deletions(-) create mode 100644 src/RSML.Abstractions.Diagnostics/AbsolutePosition.cs create mode 100644 src/RSML.Language.Lexing/Utf8/IUnsafeLexer.cs create mode 100644 src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs rename src/RSML.Language.Lexing/{ => Utf8}/Utf8Lexer.cs (72%) diff --git a/props/Defaults.props b/props/Defaults.props index 06a0955..8910879 100644 --- a/props/Defaults.props +++ b/props/Defaults.props @@ -1,7 +1,7 @@ - net10.0;net8.0 + net10.0;net9.0 true true diff --git a/src/RSML.Abstractions.Diagnostics/AbsolutePosition.cs b/src/RSML.Abstractions.Diagnostics/AbsolutePosition.cs new file mode 100644 index 0000000..fbd90ad --- /dev/null +++ b/src/RSML.Abstractions.Diagnostics/AbsolutePosition.cs @@ -0,0 +1,169 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace OceanApocalypse.RSML.Abstractions.Diagnostics; + +/// +/// An absolute reader position containing information on line and column numbers. +/// +public struct AbsolutePosition : IComparable, IEquatable, IEquatable> +{ + /// + /// A default valid position initialized at line 1 and column 1. + /// + public static readonly AbsolutePosition Default = new(1, 1); + + /// + /// The 1-based line number. + /// + public int Line + { + get; + set + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(value); + field = value; + } + } = 1; + + /// + /// The 1-based column number. + /// + public int Column + { + get; + set + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(value); + field = value; + } + } = 1; + + /// + /// Indicates whether the current instance is valid. A position is considered + /// valid if the line is positive and the column is positive as well + /// (meaning all data has been initialized). + /// + public readonly bool IsValid => Line > 0 && Column > 0; + + /// + /// Initializes a new reader position. + /// + /// The current 1-based line number. + /// The current 1-based column number. + /// + /// At least one of the parameters was negative or zero. + /// + public AbsolutePosition(int lineNumber, int columnNumber) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(lineNumber); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(columnNumber); + + Line = lineNumber; + Column = columnNumber; + } + + /// + public readonly int CompareTo(AbsolutePosition other) => (Line, Column).CompareTo((other.Line, other.Column)); + + /// + /// Moves the current position to the start of the next line. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void MoveToStartOfNextLine() + { + Line++; + Column = 1; + } + + /// + public override readonly bool Equals([NotNullWhen(true)] object? obj) => obj switch + { + AbsolutePosition pos => Equals(pos), + ValueTuple tuple => Equals(tuple), + _ => false + }; + + /// + public readonly bool Equals(AbsolutePosition other) => Line == other.Line && Column == other.Column; + + /// + /// Indicates whether the current object is equal to a tuple representation + /// of an object of the same type. + /// + /// The tuple representation. + /// True if equals. + public readonly bool Equals(ValueTuple tuple) => Line == tuple.Item1 && Column == tuple.Item2; + + /// + public override readonly int GetHashCode() => HashCode.Combine(Line, Column); + + /// + /// Returns the tuple representation of the current instance. + /// + /// The tuple representation of the instance. + public readonly (int Line, int Column) AsTuple() => new(Line, Column); + + /// + /// Indicates whether the instance to the left is equals to the instance on the right. + /// + /// One of the instances. + /// One of the instances. + /// True if equals. + public static bool operator ==(AbsolutePosition left, AbsolutePosition right) => left.Equals(right); + + /// + /// Indicates whether the instance to the left is different from the instance on the right. + /// + /// One of the instances. + /// One of the instances. + /// True if different. + public static bool operator !=(AbsolutePosition left, AbsolutePosition right) => !(left == right); + + /// + /// Indicates whether the instance to the left has a lower offset than the instance on the right. + /// + /// One of the instances. + /// One of the instances. + /// True if 's offset is less than 's. + public static bool operator <(AbsolutePosition left, AbsolutePosition right) => left.CompareTo(right) < 0; + + /// + /// Indicates whether the instance to the left has a greater offset than the instance on the right. + /// + /// One of the instances. + /// One of the instances. + /// True if 's offset is greater than 's. + public static bool operator >(AbsolutePosition left, AbsolutePosition right) => left.CompareTo(right) > 0; + + /// + /// Indicates whether the instance to the left has a lower offset than the instance on the right, + /// or if they're the same. + /// + /// One of the instances. + /// One of the instances. + /// True if 's offset is less than or equal to 's. + public static bool operator <=(AbsolutePosition left, AbsolutePosition right) => left == right || left < right; + + /// + /// Indicates whether the instance to the left has a greater offset than the instance on the right, + /// or if they're the same. + /// + /// One of the instances. + /// One of the instances. + /// True if 's offset is greater than or equal to 's. + public static bool operator >=(AbsolutePosition left, AbsolutePosition right) => left == right || left > right; + + /// + /// Throws an if the given position is invalid. + /// + /// The position to check. + /// The parameter name taken by the position. + /// Position was not initialized yet (was invalid). + public static void ThrowIfInvalid(AbsolutePosition position, string? paramName = null) + { + if (!position.IsValid) + throw new ArgumentException("Position was not initialized yet (was therefore invalid).", paramName ?? nameof(position)); + } +} \ No newline at end of file diff --git a/src/RSML.Abstractions.Diagnostics/Diagnostic.cs b/src/RSML.Abstractions.Diagnostics/Diagnostic.cs index aee4332..aab6539 100644 --- a/src/RSML.Abstractions.Diagnostics/Diagnostic.cs +++ b/src/RSML.Abstractions.Diagnostics/Diagnostic.cs @@ -10,14 +10,24 @@ namespace OceanApocalypse.RSML.Abstractions.Diagnostics; public readonly struct Diagnostic : IFormattable, IEquatable { /// - /// The start index the error relates to (inclusive). + /// The start location the error relates to (inclusive). /// - public (Index Index, int Line, int Column) Start { get; } = (0, 0, 0); + public AbsolutePosition StartLocation { get; } /// - /// The end index the error relates to (exclusive). + /// The inclusive offset at which the range starts (inclusive). /// - public (Index Index, int Line, int Column) End { get; } = (0, 0, 0); + public long StartOffset { get; } + + /// + /// The end location the error relates to (exclusive). + /// + public AbsolutePosition EndLocation { get; } + + /// + /// The exclusive offset at which the range ends (exclusive). + /// + public long EndOffset { get; } /// /// The error's code. Contains information about the category of the error. @@ -93,30 +103,33 @@ public Diagnostic(string code, string message, Severity severity) /// Creates a new diagnostic. /// The error code. - /// The inclusive start of the range. - /// The exclusive end of the range. + /// The offset at which the range starts (inclusive). + /// More details on the range's start.. + /// The offset at which the range ends (exclusive). + /// More details on the range's end. /// A brief error message detailing why it has happened. /// The error's severity. - public Diagnostic(string code, (Index idx, int line, int col) spanStart, (Index idx, int line, int col) spanEnd, string message, Severity severity) + public Diagnostic(string code, long spanStartOffset, AbsolutePosition spanStartDetails, long spanEndOffset, AbsolutePosition spanEndDetails, string message, Severity severity) { ArgumentException.ThrowIfNullOrWhiteSpace(code); ThrowIfInvalidErrorCode(code); Code = code; - Start = spanStart; - End = spanEnd; Message = message; Severity = severity; + + StartOffset = spanStartOffset; + StartLocation = spanStartDetails; + EndOffset = spanEndOffset; + EndLocation = spanEndDetails; } /// - public override bool Equals( - [NotNullWhen(true)] - object? obj - ) => obj is Diagnostic error && Equals(error); + public override bool Equals([NotNullWhen(true)] object? obj) => obj is Diagnostic error && Equals(error); /// - public bool Equals(Diagnostic other) => Message == other.Message && Code == other.Code && Severity == other.Severity && Start.Equals(other.Start) && End.Equals(other.End); + public bool Equals(Diagnostic other) => + Message == other.Message && Code == other.Code && Severity == other.Severity && StartLocation.Equals(other.StartLocation) && EndLocation.Equals(other.EndLocation); /// /// Checks if two s are equal to each other. @@ -131,13 +144,13 @@ public override bool Equals( public static bool operator !=(Diagnostic left, Diagnostic right) => !left.Equals(right); /// - public override int GetHashCode() => unchecked(HashCode.Combine(Start, End, Code, Message, Severity)); + public override int GetHashCode() => unchecked(HashCode.Combine(StartLocation, EndLocation, Code, Message, Severity)); /// /// Returns a generic string representation of the current instance. /// /// The string representation. - public override string ToString() => $"Diagnostic(Code={Code}, Start={Start}, End={End}, Message={Message}, Severity={Severity})"; + public override string ToString() => $"Diagnostic(Code={Code}, Start={StartLocation}, End={EndLocation}, Message={Message}, Severity={Severity})"; /// /// Given a format, tries to return a string that uses said format as a basis for the representation. @@ -154,7 +167,7 @@ public string ToString(string? format, IFormatProvider? formatProvider) case "I": case "INIT": case "NET": - return $"new Diagnostic(\"{Code}\", \"{Start}\", \"{End}\", \"{Message}\", {Severity})"; + return $"new Diagnostic(\"{Code}\", \"{StartLocation}\", \"{EndLocation}\", \"{Message}\", {Severity})"; case "LOG": string prefix = Severity switch @@ -166,10 +179,10 @@ public string ToString(string? format, IFormatProvider? formatProvider) _ => "" }; - if (Start.Line == End.Line) - return $"[{prefix}{Code}] @ L{Start.Line + 1},C({Start.Column + 1}..{End.Column + 1}) : {Message}"; + if (StartLocation.Line == EndLocation.Line) + return $"[{prefix}{Code}] @ L{StartLocation.Line + 1},C({StartLocation.Column + 1}..{EndLocation.Column + 1}) : {Message}"; - return $"[{prefix}{Code}] @ L({Start.Line + 1}..{End.Line + 1}),C({Start.Column + 1}..{End.Column + 1}) : {Message}"; + return $"[{prefix}{Code}] @ L({StartLocation.Line + 1}..{EndLocation.Line + 1}),C({StartLocation.Column + 1}..{EndLocation.Column + 1}) : {Message}"; case "JSON": return $$""" @@ -177,20 +190,14 @@ public string ToString(string? format, IFormatProvider? formatProvider) "errorCode": "{{Code}}", "range": [ { - "index": { - "value": {{Start.Index.Value}}, - "isFromEnd": {{Start.Index.IsFromEnd}} - }, - "line": {{Start.Line}}, - "column": {{Start.Column}} + "offset": {{StartOffset}}, + "line": {{StartLocation.Line}}, + "column": {{StartLocation.Column}} }, { - "index": { - "value": {{End.Index.Value}}, - "isFromEnd": {{End.Index.IsFromEnd}} - }, - "line": {{End.Line}}, - "column": {{End.Column}} + "offset": {{EndOffset}}, + "line": {{EndLocation.Line}}, + "column": {{EndLocation.Column}} } ] } diff --git a/src/RSML.Abstractions/Extensions.cs b/src/RSML.Abstractions/Extensions.cs index f28c75a..49bd254 100644 --- a/src/RSML.Abstractions/Extensions.cs +++ b/src/RSML.Abstractions/Extensions.cs @@ -68,6 +68,13 @@ public static class Extensions /// True if the character is an ASCII digit. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsAsciiDigit() => item is >= 48 and <= 57; // 48 is '0' and 57 is '9' + + /// + /// Checks if a given character falls under the ASCII category. + /// + /// True if the character is ASCII. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsAscii() => item is > 127; } extension(IImmutableList strings) diff --git a/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs b/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs index b87355e..6834443 100644 --- a/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs +++ b/src/RSML.Language.Lexing/Diagnostics/LexerErrorCodes.cs @@ -8,4 +8,5 @@ internal static class LexerErrorCodes public const string FailedToIdentifyKeyword = "RL0003"; public const string FailedToIdentifyPunctuation = "RL0004"; public const string ExpectedStdIdentifier = "RL0005"; + public const string InvalidData = "RL0006"; } diff --git a/src/RSML.Language.Lexing/ILexer.cs b/src/RSML.Language.Lexing/ILexer.cs index d33e10c..f177ca0 100644 --- a/src/RSML.Language.Lexing/ILexer.cs +++ b/src/RSML.Language.Lexing/ILexer.cs @@ -1,29 +1,24 @@ -using System; -using System.Buffers; using System.Collections.Generic; -using OceanApocalypse.RSML.Abstractions.Diagnostics; using OceanApocalypse.RSML.Abstractions.Toolchain; using OceanApocalypse.RSML.Language.Lexing.Tokens; - namespace OceanApocalypse.RSML.Language.Lexing; /// -/// Represents a lexer for RSML. +/// Represents a lexer tasked with tokenizing RSML code. /// -public interface ILexer : IToolchainComponent - where TInput : unmanaged, IEquatable +public interface ILexer : IToolchainComponent { - /// - /// Tokenizes a source passed to the lexer. + /// + /// Tokenizes a string passed to the lexer. /// /// The tokens. - IEnumerable Lex(ReadOnlySequence data); + IEnumerable Lex(string? data); - /// - /// Returns the next token. + /// + /// Tokenizes an array of characters passed to the lexer. /// - /// The next token. - Result GetNextToken(ref SequenceReader reader); + /// The tokens. + IEnumerable Lex(char[] data); } diff --git a/src/RSML.Language.Lexing/Utf8/IUnsafeLexer.cs b/src/RSML.Language.Lexing/Utf8/IUnsafeLexer.cs new file mode 100644 index 0000000..214665d --- /dev/null +++ b/src/RSML.Language.Lexing/Utf8/IUnsafeLexer.cs @@ -0,0 +1,19 @@ +using System; + +using OceanApocalypse.RSML.Language.Lexing.Tokens; + +namespace OceanApocalypse.RSML.Language.Lexing.Utf8; + +/// +/// Represents a lexer with attributes or methods that utilize the unsafe context. +/// +[CLSCompliant(false)] +public unsafe interface IUnsafeLexer : ILexer +{ + /// + /// Tokenizes an array of bytes passed to the lexer, with UTF-8 encoding. + /// + /// The tokens. + [CLSCompliant(false)] + Token* Lex(byte* data, int charCount); +} diff --git a/src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs b/src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs new file mode 100644 index 0000000..bee2155 --- /dev/null +++ b/src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs @@ -0,0 +1,42 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO.Pipes; +using System.Threading; +using System.Threading.Tasks; + +using OceanApocalypse.RSML.Abstractions.Diagnostics; +using OceanApocalypse.RSML.Abstractions.Toolchain; +using OceanApocalypse.RSML.Language.Lexing.Tokens; + + +namespace OceanApocalypse.RSML.Language.Lexing.Utf8; + +/// +/// Represents a UTF-8 lexer for RSML. +/// +public interface IUtf8Lexer : ILexer + where TInput : unmanaged, IEquatable +{ + /// + /// Tokenizes a source passed to the lexer. + /// + /// The tokens. + IEnumerable Lex(ReadOnlySequence data); + + /// + /// Tokenizes a source passed to the lexer asynchronously. + /// + /// The source as a byte stream + /// A cancellation token. + /// The tokens. + Task> LexAsync(PipeStream stream, CancellationToken? cancellationToken = default); + + /// + /// Returns the next token. + /// + /// The reader whose data to read. + /// The current expected position. + /// The next token. + Result GetNextToken(ref SequenceReader reader, ref AbsolutePosition currentPosition); +} diff --git a/src/RSML.Language.Lexing/Utf8Lexer.cs b/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs similarity index 72% rename from src/RSML.Language.Lexing/Utf8Lexer.cs rename to src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs index c250c71..faadc89 100644 --- a/src/RSML.Language.Lexing/Utf8Lexer.cs +++ b/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Data; +using System.IO.Pipes; +using System.Threading; +using System.Threading.Tasks; using OceanApocalypse.RSML.Abstractions; using OceanApocalypse.RSML.Abstractions.Diagnostics; @@ -11,7 +14,7 @@ using OceanApocalypse.RSML.Language.Lexing.Diagnostics; using OceanApocalypse.RSML.Language.Lexing.Tokens; -namespace OceanApocalypse.RSML.Language.Lexing; +namespace OceanApocalypse.RSML.Language.Lexing.Utf8; /// /// An implementation of a RSML lexer backed by a given UTF-8 buffer. @@ -21,7 +24,7 @@ namespace OceanApocalypse.RSML.Language.Lexing; /// /// A collector with all the diagnostics that were and will be emitted. /// Configurations for the toolchain components. -public class Utf8Lexer(DiagnosticCollector diagnosticCollector, ToolchainConfiguration? configuration = null) : ILexer +public class Utf8Lexer(DiagnosticCollector diagnosticCollector, ToolchainConfiguration? configuration = null) : IUtf8Lexer { private bool isDisposed; private bool wasUsed; @@ -42,24 +45,37 @@ public class Utf8Lexer(DiagnosticCollector diagnosticCollector, ToolchainConfigu /// ::: /// /// - public Result GetNextToken(ref SequenceReader reader) + public Result GetNextToken(ref SequenceReader reader, ref AbsolutePosition currentPosition) { wasUsed = true; + AbsolutePosition.ThrowIfInvalid(currentPosition); + + var startLoc = reader.Consumed; SkipWhitespaceAndComments(ref reader); - long startLoc = reader.Consumed; if (reader.End || !reader.TryPeek(out byte b)) - return Result.Success(new Token(TokenKind.Eof, startLoc, 0L)); + return Result.Success(new Token(TokenKind.Eof, startLoc, startLoc)); + + if (!b.IsAscii()) + { + return Result.Failure(new( + LexerErrorCodes.InvalidData, + startLoc, currentPosition, + startLoc, currentPosition, + "Expected an ASCII character but received a non-ASCII character.", + Severity.Error + )); + } char c = (char)b; // strings if (c == '"') - return ScanStringLiteral(ref reader, startLoc); + return ScanStringLiteral(ref reader, startLoc, ref currentPosition); // number literals if (b.IsAsciiDigit()) - return ScanNumber(ref reader, b, startLoc); + return ScanNumber(ref reader, startChar: b, startLoc, ref currentPosition); // identifiers and keywords if (b.IsAsciiLetter() || c == '_') @@ -80,7 +96,7 @@ public Result GetNextToken(ref SequenceReader reader) // todo: check for comments if Configuration.EmitComments is enabled return Result.Failure(new( - LexerErrorCodes.FailedToLexToken, + code: LexerErrorCodes.FailedToLexToken, "Tried all possible token logic paths, but none was true. This likely means you used a character not recognized by the lexer," + "but it may also mean the lexer is mal-functioning.", Severity.Critical @@ -117,8 +133,9 @@ public IEnumerable Lex(ReadOnlySequence data) ); } - private static Result ScanNumber(ref SequenceReader reader, byte startChar, long startLoc) + private static Result ScanNumber(ref SequenceReader reader, byte startChar, long startLoc, ref AbsolutePosition position) { + var startPos = position; const byte underscore = (byte)'_'; const byte dot = (byte)'.'; @@ -128,6 +145,18 @@ private static Result ScanNumber(ref SequenceReader reader, byte st do { reader.Advance(1); + position.Column++; + + if (!b.IsAscii()) + { + return Result.Failure(new( + LexerErrorCodes.InvalidData, + startLoc, startPos, + startLoc, position, + "Expected an ASCII character but received a non-ASCII character.", + Severity.Error + )); + } if (b == dot) hasDotSeparator = true; @@ -137,38 +166,51 @@ private static Result ScanNumber(ref SequenceReader reader, byte st return Result.Success(new Token(TokenKind.NumericLiteral, startLoc, reader.Consumed - startLoc)); } - // todo: fix the method below - private Result ScanStringLiteral(ref SequenceReader reader, long startLoc) + private static Result ScanStringLiteral(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) { - cursor++; + var startPos = position; bool escaping = false; - while (cursor < Sequence.Length) + if (reader.End) + { + return Result.Failure(new( + LexerErrorCodes.UnterminatedStringLiteral, + startLoc, startPos, + reader.Consumed, position, + "A string literal must begin and end in the same line.", + Severity.Error + )); + } + + reader.Advance(1); + position.Column++; + + while (!reader.End && reader.TryPeek(out byte b)) { - if (Sequence[cursor].IsNewline()) + reader.Advance(1); + position.Column++; + + if (b.IsAsciiNewline()) { + position.MoveToStartOfNextLine(); + return Result.Failure(new( LexerErrorCodes.UnterminatedStringLiteral, - Sequence.GetLocationDetails((Index)startLoc), - Sequence.GetLocationDetails((Index)cursor), + startLoc, startPos, + reader.Consumed, position, "A string literal must begin and end in the same line.", Severity.Error )); } - if (Sequence[cursor] == '"' && !escaping) + if (b == '"' && !escaping) break; - if (Sequence[cursor] == '\\') + if (b == '\\') escaping = !escaping; - - cursor++; } - if (cursor < Sequence.Length) - cursor++; // skip end quote if anything beyond it - - return Result.Success(new Token(TokenKind.StringLiteral, null, startLoc..cursor)); + return Result.Success(new Token(TokenKind.StringLiteral, startLoc, reader.Consumed - startLoc)); } // todo: fix the method below @@ -286,4 +328,22 @@ protected virtual void Dispose(bool disposing) isDisposed = true; } + + /// + /// Checks if the given position matches the cursor position in the + /// reader. This does not check if the line and column numbers are correct. + /// + /// The reader. + /// The expected current position. + /// True if the given position matches the reader's cursor position, in offset. + public static bool IsCorrectPosition(in SequenceReader reader, AbsolutePosition position) => position.Offset == reader.Consumed; + + // todo: implement the method below + public Task> LexAsync(PipeStream stream, CancellationToken? cancellationToken = null) => throw new NotImplementedException(); + + // todo: implement the method below + public IEnumerable Lex(string? data) => throw new NotImplementedException(); + + // todo: implement the method below + public IEnumerable Lex(char[] data) => throw new NotImplementedException(); } From 5f775a8c17eebaf9d18afa3d2948660ed35995c4 Mon Sep 17 00:00:00 2001 From: Matthew Date: Wed, 12 Aug 2026 17:05:53 +0100 Subject: [PATCH 10/11] feat(lexer)!: Fully implement the UTF-8 lexer [3/3] Context: Refactor the lexer for performance (using UTF-8 and ASCII extensions instead of UTF-16 string methods) Code is no longer broken. Signed-off-by: Matthew Refs: #49 --- benchmarks/Program.cs | 2 - .../AssemblyInfo.cs | 1 - src/RSML.Abstractions.Panic/AssemblyInfo.cs | 1 - .../AssemblyInfo.cs | 1 - .../IToolchainComponent.cs | 1 - src/RSML.Abstractions/AssemblyInfo.cs | 1 - src/RSML.Abstractions/Extensions.cs | 69 +++++++-- src/RSML.Abstractions/GlobalSuppressions.cs | 2 - src/RSML.Language.Lexing/AssemblyInfo.cs | 1 - src/RSML.Language.Lexing/Tokens/Token.cs | 103 ++++++++----- src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs | 12 -- src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs | 139 ++++++++++-------- src/RSML.Language.Parsing/Parser.cs | 6 +- .../Interpreter.cs | 5 +- 14 files changed, 210 insertions(+), 134 deletions(-) diff --git a/benchmarks/Program.cs b/benchmarks/Program.cs index a84de00..ee20d84 100644 --- a/benchmarks/Program.cs +++ b/benchmarks/Program.cs @@ -1,5 +1,3 @@ -using BenchmarkDotNet.Running; - namespace OceanApocalypse.RSML.Benchmarks; internal sealed class Program diff --git a/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs b/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs index 26fce2e..1b4601d 100644 --- a/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs +++ b/src/RSML.Abstractions.Diagnostics/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // In SDK-style projects such as this one, several assembly attributes that were historically diff --git a/src/RSML.Abstractions.Panic/AssemblyInfo.cs b/src/RSML.Abstractions.Panic/AssemblyInfo.cs index 26fce2e..1b4601d 100644 --- a/src/RSML.Abstractions.Panic/AssemblyInfo.cs +++ b/src/RSML.Abstractions.Panic/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // In SDK-style projects such as this one, several assembly attributes that were historically diff --git a/src/RSML.Abstractions.Toolchain/AssemblyInfo.cs b/src/RSML.Abstractions.Toolchain/AssemblyInfo.cs index 26fce2e..1b4601d 100644 --- a/src/RSML.Abstractions.Toolchain/AssemblyInfo.cs +++ b/src/RSML.Abstractions.Toolchain/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // In SDK-style projects such as this one, several assembly attributes that were historically diff --git a/src/RSML.Abstractions.Toolchain/IToolchainComponent.cs b/src/RSML.Abstractions.Toolchain/IToolchainComponent.cs index b99968d..85abbfa 100644 --- a/src/RSML.Abstractions.Toolchain/IToolchainComponent.cs +++ b/src/RSML.Abstractions.Toolchain/IToolchainComponent.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; namespace OceanApocalypse.RSML.Abstractions.Toolchain; diff --git a/src/RSML.Abstractions/AssemblyInfo.cs b/src/RSML.Abstractions/AssemblyInfo.cs index 26fce2e..1b4601d 100644 --- a/src/RSML.Abstractions/AssemblyInfo.cs +++ b/src/RSML.Abstractions/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // In SDK-style projects such as this one, several assembly attributes that were historically diff --git a/src/RSML.Abstractions/Extensions.cs b/src/RSML.Abstractions/Extensions.cs index 49bd254..5b60aa9 100644 --- a/src/RSML.Abstractions/Extensions.cs +++ b/src/RSML.Abstractions/Extensions.cs @@ -1,6 +1,8 @@ using System; -using System.Collections.Immutable; +using System.Buffers; +using System.Collections.Frozen; using System.Runtime.CompilerServices; +using System.Text; namespace OceanApocalypse.RSML.Abstractions; @@ -9,6 +11,7 @@ namespace OceanApocalypse.RSML.Abstractions; /// public static class Extensions { + private readonly static UTF8Encoding exceptionlessUtf8Encoding = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false); private const byte UpperLowerDiffBit = 0b_0010_0000; // 0x20, binary seems best suited for this tho ngl #region ASCII Characters @@ -77,23 +80,69 @@ public static class Extensions public bool IsAscii() => item is > 127; } - extension(IImmutableList strings) + extension(ReadOnlySequence sequence) { /// - /// Checks if an immutable array of strings contains a given character span. + /// Safely decodes a UTF-8 sequence to a UTF-16 character span. /// - /// The span to check for. - /// The comparison mode to apply. + /// The destination span. + /// The UTF-8 encoding to use. + /// The amount of characters written. + public int SafelyDecodeToCharacterSpan(Span destination, UTF8Encoding? encoding = null) + { + int totalCharsWritten = 0; + Decoder decoder = (encoding ?? exceptionlessUtf8Encoding).GetDecoder(); + + foreach (var segment in sequence) + { + ReadOnlySpan span = segment.Span; + decoder.Convert(span, destination[totalCharsWritten..], false, out _, out int charsWritten, out _); + totalCharsWritten += charsWritten; + } + + decoder.Convert([], destination[totalCharsWritten..], true, out _, out int flushCharsWritten, out _); + totalCharsWritten += flushCharsWritten; + return totalCharsWritten; + } + } + + extension(FrozenSet stringSet) + { + /// + /// Checks if an immutable array of UTF-16 strings contains a given UTF-8 span. + /// + /// The span to check for. + /// The threshold that, when exceeded, ensures the code falls back to using arrays to avoid overflows. /// True if found. - public bool Contains(ReadOnlySpan span, StringComparison comparisonType = StringComparison.Ordinal) + public bool ContainsUtf8(ReadOnlySequence seq, int stackAllocThreshold = 256) { - foreach (string @string in strings) + if (stringSet.Count == 0 || seq.IsEmpty) + return false; // we fail fast over here + + var lookup = stringSet.GetAlternateLookup>(); // lookup spans, not strings to avoid heap allocs obviously + long length = seq.Length; + + if (length <= stackAllocThreshold) // the seq is small, we can use a span directly :) { - if (!span.Equals(@string, comparisonType)) - return false; + Span charBuffer = stackalloc char[(int)length]; + int charsWritten = seq.SafelyDecodeToCharacterSpan(charBuffer, exceptionlessUtf8Encoding); + return lookup.Contains(charBuffer[..charsWritten]); // make sure to slice to avoid reading garbage data broski } + else + { + char[] pooledArray = ArrayPool.Shared.Rent((int)length); // we poolin' over here (direct span would be stack overflow) - return true; + try + { + Span charBuffer = pooledArray.AsSpan(0, (int)length); // get a span out of it, dum dum (we limit to length because sometimes it might pool larger arrays) + int charsWritten = seq.SafelyDecodeToCharacterSpan(charBuffer, exceptionlessUtf8Encoding); + return lookup.Contains(charBuffer[..charsWritten]); // make sure to slice to avoid reading garbage data brosquito + } + finally + { + ArrayPool.Shared.Return(pooledArray); // we return to avoid leaking + } + } } } } diff --git a/src/RSML.Abstractions/GlobalSuppressions.cs b/src/RSML.Abstractions/GlobalSuppressions.cs index a1fe9a6..3dc76b4 100644 --- a/src/RSML.Abstractions/GlobalSuppressions.cs +++ b/src/RSML.Abstractions/GlobalSuppressions.cs @@ -3,5 +3,3 @@ // Project-level suppressions either have no target or are given // a specific target and scoped to a namespace, type, member, etc. -using System.Diagnostics.CodeAnalysis; - diff --git a/src/RSML.Language.Lexing/AssemblyInfo.cs b/src/RSML.Language.Lexing/AssemblyInfo.cs index 26fce2e..1b4601d 100644 --- a/src/RSML.Language.Lexing/AssemblyInfo.cs +++ b/src/RSML.Language.Lexing/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // In SDK-style projects such as this one, several assembly attributes that were historically diff --git a/src/RSML.Language.Lexing/Tokens/Token.cs b/src/RSML.Language.Lexing/Tokens/Token.cs index b58b1a6..d301136 100644 --- a/src/RSML.Language.Lexing/Tokens/Token.cs +++ b/src/RSML.Language.Lexing/Tokens/Token.cs @@ -1,7 +1,11 @@ using System; +using System.Buffers; +using System.Collections.Frozen; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using OceanApocalypse.RSML.Abstractions; + namespace OceanApocalypse.RSML.Language.Lexing.Tokens; @@ -14,36 +18,59 @@ namespace OceanApocalypse.RSML.Language.Lexing.Tokens; [StructLayout(LayoutKind.Sequential)] public record struct Token(TokenKind Kind, long StartOffset, long Length) { + /// + /// Any keyword with anything that exceeds this length immediately skips the keyword check. + /// + private const int MaxKeywordLength = 32; + /// /// Empty token. Used when something goes wrong. /// public readonly static Token Empty = new(TokenKind.Unknown, 0L, 0L); + /// + /// The available keywords and keyword modifiers. + /// + public readonly static FrozenSet Keywords = [ + "return", "if", "requires", "end", "previous", "region", "let", + "mut", "fn", "exec", "type", "as", "struct", + "class", "interface" // these 2 are reserved + ]; + /// /// Gets the token kind that applies to the given keyword or keyword modifier. /// - /// The keyword or modifier. + /// The keyword or modifier sequence. /// The matching token kind. - public static TokenKind GetKeywordKind(scoped ReadOnlySpan keyword) => keyword switch + public static TokenKind GetKeywordKind(ReadOnlySequence sequence) { - // keywords - "as" => TokenKind.AsKeyword, - "end" => TokenKind.EndKeyword, - "if" => TokenKind.IfKeyword, - "let" => TokenKind.LetKeyword, - "region" => TokenKind.RegionKeyword, - "requires" => TokenKind.RequiresKeyword, - "return" => TokenKind.ReturnKeyword, - "struct" => TokenKind.StructKeyword, - "type" => TokenKind.TypeKeyword, - - // modifiers - "fn" => TokenKind.FunctionModifier, - "mut" => TokenKind.MutableModifier, - "previous" => TokenKind.PreviousModifier, - - _ => TokenKind.Unknown, - }; + if (sequence.Length > MaxKeywordLength) + return TokenKind.Unknown; // we love failing fast + + Span keywordBuffer = stackalloc char[MaxKeywordLength]; + int keywordLength = sequence.SafelyDecodeToCharacterSpan(keywordBuffer); + + return keywordBuffer[..keywordLength] switch + { + // keywords + "as" => TokenKind.AsKeyword, + "end" => TokenKind.EndKeyword, + "if" => TokenKind.IfKeyword, + "let" => TokenKind.LetKeyword, + "region" => TokenKind.RegionKeyword, + "requires" => TokenKind.RequiresKeyword, + "return" => TokenKind.ReturnKeyword, + "struct" => TokenKind.StructKeyword, + "type" => TokenKind.TypeKeyword, + + // modifiers + "fn" => TokenKind.FunctionModifier, + "mut" => TokenKind.MutableModifier, + "previous" => TokenKind.PreviousModifier, + + _ => TokenKind.Unknown, + }; + } /// /// Gets the token kind that applies to the given punctuation symbol. @@ -51,35 +78,35 @@ public record struct Token(TokenKind Kind, long StartOffset, long Length) /// The punctuation character. /// /// The character that follows . - /// Set to null if out of bounds. Default is null. + /// Set to 0 if out of bounds/not punctuation. /// /// - public static TokenKind GetPunctuationKind(char punctuation, char? peekedChar = null) => punctuation switch + public static TokenKind GetPunctuationKind(byte punctuation, byte peekedChar) => punctuation switch { // math operations - '+' => TokenKind.Plus, - '-' => TokenKind.Minus, - '*' => TokenKind.Star, - '/' => TokenKind.Slash, + (byte)'+' => TokenKind.Plus, + (byte)'-' => TokenKind.Minus, + (byte)'*' => TokenKind.Star, + (byte)'/' => TokenKind.Slash, // equality - '=' when peekedChar is '=' => TokenKind.EqualToOperator, - '!' when peekedChar is '=' => TokenKind.NotEqualToOperator, - '>' when peekedChar is '=' => TokenKind.GreaterThanOrEqualToOperator, - '<' when peekedChar is '=' => TokenKind.LessThanOrEqualToOperator, - '>' => TokenKind.GreaterThanOperator, - '<' => TokenKind.LessThanOperator, + (byte)'=' when peekedChar is (byte)'=' => TokenKind.EqualToOperator, + (byte)'!' when peekedChar is (byte)'=' => TokenKind.NotEqualToOperator, + (byte)'>' when peekedChar is (byte)'=' => TokenKind.GreaterThanOrEqualToOperator, + (byte)'<' when peekedChar is (byte)'=' => TokenKind.LessThanOrEqualToOperator, + (byte)'>' => TokenKind.GreaterThanOperator, + (byte)'<' => TokenKind.LessThanOperator, // boolean logic - '&' when peekedChar is '&' => TokenKind.LogicAndOperator, - '|' when peekedChar is '|' => TokenKind.LogicOrOperator, - '!' => TokenKind.NotOperator, + (byte)'&' when peekedChar is (byte)'&' => TokenKind.LogicAndOperator, + (byte)'|' when peekedChar is (byte)'|' => TokenKind.LogicOrOperator, + (byte)'!' => TokenKind.NotOperator, - '=' => TokenKind.AssignmentOperator, + (byte)'=' => TokenKind.AssignmentOperator, // reserved - '&' => TokenKind.Unknown, - '|' => TokenKind.Unknown, + (byte)'&' => TokenKind.Unknown, + (byte)'|' => TokenKind.Unknown, _ => TokenKind.Unknown }; diff --git a/src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs b/src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs index bee2155..7d0f2ae 100644 --- a/src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs +++ b/src/RSML.Language.Lexing/Utf8/IUtf8Lexer.cs @@ -1,12 +1,8 @@ using System; using System.Buffers; using System.Collections.Generic; -using System.IO.Pipes; -using System.Threading; -using System.Threading.Tasks; using OceanApocalypse.RSML.Abstractions.Diagnostics; -using OceanApocalypse.RSML.Abstractions.Toolchain; using OceanApocalypse.RSML.Language.Lexing.Tokens; @@ -24,14 +20,6 @@ public interface IUtf8Lexer : ILexer /// The tokens. IEnumerable Lex(ReadOnlySequence data); - /// - /// Tokenizes a source passed to the lexer asynchronously. - /// - /// The source as a byte stream - /// A cancellation token. - /// The tokens. - Task> LexAsync(PipeStream stream, CancellationToken? cancellationToken = default); - /// /// Returns the next token. /// diff --git a/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs b/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs index faadc89..09862b0 100644 --- a/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs +++ b/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs @@ -1,11 +1,8 @@ using System; using System.Buffers; using System.Collections.Generic; -using System.Collections.Immutable; using System.Data; -using System.IO.Pipes; -using System.Threading; -using System.Threading.Tasks; +using System.Text; using OceanApocalypse.RSML.Abstractions; using OceanApocalypse.RSML.Abstractions.Diagnostics; @@ -79,19 +76,23 @@ public Result GetNextToken(ref SequenceReader reader, ref AbsoluteP // identifiers and keywords if (b.IsAsciiLetter() || c == '_') - return ScanIdentifierOrKeyword(ref reader, startLoc); + return ScanIdentifierOrKeyword(ref reader, startLoc, ref currentPosition); // standard library identifiers if (c == '$') - return ScanStdIdentifier(ref reader, startLoc); + return ScanStdIdentifier(ref reader, startLoc, ref currentPosition); // member access notation if (c == '.') - return Result.Success(new Token(TokenKind.MemberAccess, startLoc, 1L)); + { + reader.Advance(1); + currentPosition.Column++; + return Result.Success(new Token(TokenKind.MemberAccess, startLoc, 1)); + } // punctuation if (b.IsRsmlPunctuation()) - return ScanPunctuation(ref reader, startLoc); + return ScanPunctuation(ref reader, startLoc, ref currentPosition); // todo: check for comments if Configuration.EmitComments is enabled @@ -109,10 +110,18 @@ public IEnumerable Lex(ReadOnlySequence data) wasUsed = true; int failedRuns = 0; + var reader = new SequenceReader(data); + var position = AbsolutePosition.Default; + + if (!position.IsValid) + { + position.Line = 1; + position.Column = 1; + } + while (Configuration.MaximumAllowedFailuresPerComponent <= 0 || failedRuns < Configuration.MaximumAllowedFailuresPerComponent) { - // todo: create reader here - var token = GetNextToken(ref reader); + var token = GetNextToken(ref reader, ref position); if (token.IsError) { @@ -136,8 +145,6 @@ public IEnumerable Lex(ReadOnlySequence data) private static Result ScanNumber(ref SequenceReader reader, byte startChar, long startLoc, ref AbsolutePosition position) { var startPos = position; - const byte underscore = (byte)'_'; - const byte dot = (byte)'.'; byte b = startChar; bool hasDotSeparator = false; @@ -158,10 +165,10 @@ private static Result ScanNumber(ref SequenceReader reader, byte st )); } - if (b == dot) + if (b == '.') hasDotSeparator = true; - } while (!reader.End && reader.TryPeek(out b) && (b.IsAsciiDigit() || b == underscore || (b == dot && !hasDotSeparator))); + } while (!reader.End && reader.TryPeek(out b) && (b.IsAsciiDigit() || b == '_' || (b == '.' && !hasDotSeparator))); return Result.Success(new Token(TokenKind.NumericLiteral, startLoc, reader.Consumed - startLoc)); } @@ -213,43 +220,50 @@ private static Result ScanStringLiteral(ref SequenceReader reader, return Result.Success(new Token(TokenKind.StringLiteral, startLoc, reader.Consumed - startLoc)); } - // todo: fix the method below - private Result ScanStdIdentifier(ref SequenceReader reader, long startLoc) + private static Result ScanStdIdentifier(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) { - // this points to h in $helloWorld broski - int afterStdSymbolIndex = ++cursor; // we also skip past it to avoid extra checks in while loop + AbsolutePosition startPos = position; + + do + { + reader.Advance(1); + position.Column++; - while (cursor < Sequence.Length && (Char.IsAsciiLetterOrDigit(Sequence[cursor]) || Sequence[cursor] == '_')) - cursor++; + } while (!reader.End && reader.TryPeek(out byte b) && (b.IsAsciiLetter() || b.IsAsciiDigit() || b == '_')); - return cursor == afterStdSymbolIndex + return reader.Consumed == startLoc + 1 ? Result.Failure(new( LexerErrorCodes.ExpectedStdIdentifier, - Sequence.GetLocationDetails((Index)startLoc), - Sequence.GetLocationDetails((Index)cursor), + startLoc, startPos, + reader.Consumed, position, "Expected a standard library identifier, yet there was no valid identifier after the $ symbol.", Severity.Error )) - : Result.Success(new Token(TokenKind.StandardLibraryIdentifier, null, startLoc..cursor)); + : Result.Success(new Token(TokenKind.StandardLibraryIdentifier, startLoc, reader.Consumed - startLoc)); } - // todo: fix the method below - private Result ScanIdentifierOrKeyword(ref SequenceReader reader, long startLoc) + private static Result ScanIdentifierOrKeyword(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) { - while (cursor < Sequence.Length && (Char.IsAsciiLetterOrDigit(Sequence[cursor]) || Sequence[cursor] == '_')) - cursor++; + var startPos = position; - Range range = startLoc..cursor; + while (!reader.End && reader.TryPeek(out byte b) && (b.IsAsciiLetter() || b.IsAsciiDigit() || b == '_')) + { + reader.Advance(1); + position.Column++; + } + + int sliceLength = (int)(reader.Consumed - startLoc); + var slice = reader.Sequence.Slice(startLoc, sliceLength); - if (Keywords.Contains(Sequence[range])) + if (Token.Keywords.ContainsUtf8(slice)) { - var token = new Token(Token.GetKeywordKind(Sequence[range]), null, range); // is keyword + var token = new Token(Token.GetKeywordKind(slice), startLoc, sliceLength); // is keyword return token.Kind == TokenKind.Unknown ? Result.Failure(new( LexerErrorCodes.FailedToIdentifyKeyword, - Sequence.GetLocationDetails(startLoc), - Sequence.GetLocationDetails(cursor), + startLoc, startPos, + reader.Consumed, position, "Despite identifying the object in question as a keyword, the lexer failed to resolve exactly which keyword it was." + "This likely means the keyword in question is reserved for future use, but isn't implemented yet.", Severity.Error @@ -258,27 +272,40 @@ private Result ScanIdentifierOrKeyword(ref SequenceReader reader, l } else { - return Result.Success(new Token(TokenKind.Identifier, null, range)); // is identifier + return Result.Success(new Token(TokenKind.Identifier, startLoc, sliceLength)); // is identifier } } - // todo: fix the method below - private Result ScanPunctuation(ref SequenceReader reader, long startLoc) + private static Result ScanPunctuation(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) { - char c = Sequence[cursor]; - char? peeked = cursor + 1 >= Sequence.Length ? null : Sequence[++cursor]; // dont error out if out of bounds - TokenKind kind = Token.GetPunctuationKind(c, peeked); + var startPos = position; + + reader.TryRead(out byte first); // will always work and will always be ASCII + position.Column++; + + var successful = reader.TryPeek(out byte second); + + if (successful && !second.IsAscii()) + successful = false; + + if (successful) + { + reader.Advance(1); + position.Column++; + } + + TokenKind kind = Token.GetPunctuationKind(first, successful ? second : (byte)0); return kind == TokenKind.Unknown ? Result.Failure(new( LexerErrorCodes.FailedToIdentifyPunctuation, - Sequence.GetLocationDetails(startLoc), - Sequence.GetLocationDetails(peeked is null ? cursor - 1 : cursor), + startLoc, startPos, + reader.Consumed, position, "Despite identifying the object in question as punctuation, the lexer failed to resolve exactly which punctuation it was." + "This might mean the punctuation in question is reserved for future use, and not implemented yet.", Severity.Error )) - : Result.Success(new Token(kind, null, startLoc..cursor)); + : Result.Success(new Token(kind, startLoc, reader.Consumed - startLoc)); } private void SkipWhitespaceAndComments(ref SequenceReader reader) @@ -329,21 +356,17 @@ protected virtual void Dispose(bool disposing) isDisposed = true; } - /// - /// Checks if the given position matches the cursor position in the - /// reader. This does not check if the line and column numbers are correct. - /// - /// The reader. - /// The expected current position. - /// True if the given position matches the reader's cursor position, in offset. - public static bool IsCorrectPosition(in SequenceReader reader, AbsolutePosition position) => position.Offset == reader.Consumed; - - // todo: implement the method below - public Task> LexAsync(PipeStream stream, CancellationToken? cancellationToken = null) => throw new NotImplementedException(); - - // todo: implement the method below - public IEnumerable Lex(string? data) => throw new NotImplementedException(); + /// + public IEnumerable Lex(string? data) + { + ArgumentException.ThrowIfNullOrEmpty(data); + return Lex(new ReadOnlySequence(Encoding.Default.GetBytes(data))); + } - // todo: implement the method below - public IEnumerable Lex(char[] data) => throw new NotImplementedException(); + /// + public IEnumerable Lex(char[] data) + { + ArgumentNullException.ThrowIfNull(data); + return Lex(new ReadOnlySequence(Encoding.Default.GetBytes(data))); + } } diff --git a/src/RSML.Language.Parsing/Parser.cs b/src/RSML.Language.Parsing/Parser.cs index 77dfe04..ae55cbd 100644 --- a/src/RSML.Language.Parsing/Parser.cs +++ b/src/RSML.Language.Parsing/Parser.cs @@ -1,6 +1,6 @@ using System; -using OceanApocalypse.RSML.Abstractions; +using OceanApocalypse.RSML.Abstractions.Toolchain; namespace OceanApocalypse.RSML.Language.Parsing; @@ -14,7 +14,7 @@ public abstract class Parser : IParser private bool isDisposed; /// - public ToolchainConfigurations Configuration { get; protected set; } + ToolchainConfiguration IToolchainComponent.Configuration => throw new NotImplementedException(); /// public void Dispose() @@ -24,7 +24,7 @@ public void Dispose() } /// - public virtual void Inject(ToolchainConfigurations configuration) => throw new NotImplementedException(); + public void Inject(ToolchainConfiguration configuration) => throw new NotImplementedException(); /// /// Disposes of both managed and unmanaged resources. diff --git a/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs b/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs index 183d69e..88fa9ca 100644 --- a/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs +++ b/src/RSML.Toolchain.Extensibility.Execution/Interpreter.cs @@ -1,6 +1,5 @@ using System; -using OceanApocalypse.RSML.Abstractions; using OceanApocalypse.RSML.Abstractions.Toolchain; namespace OceanApocalypse.RSML.Toolchain.Extensibility.Execution; @@ -18,7 +17,7 @@ public abstract class Interpreter : IToolchainComponent public bool IsMutable { get; protected set; } = true; /// - public ToolchainConfigurations Configuration { get; protected set; } + ToolchainConfiguration IToolchainComponent.Configuration => throw new NotImplementedException(); /// public void Dispose() @@ -31,7 +30,7 @@ public void Dispose() public void Freeze() => IsMutable = false; /// - public void Inject(ToolchainConfigurations configuration) => throw new NotImplementedException(); + public void Inject(ToolchainConfiguration configuration) => throw new NotImplementedException(); /// /// Disposes of both managed and unmanaged resources. From 5af2fe6378e4aa764b2273b196c993d8ed834b60 Mon Sep 17 00:00:00 2001 From: Matthew Date: Wed, 12 Aug 2026 20:36:27 +0100 Subject: [PATCH 11/11] fix(lexer): Fix incorrect lexer behaviors regarding yield usage and comment scanning --- .../GlobalSuppressions.cs | 2 +- src/RSML.Language.Lexing/Tokens/TokenKind.cs | 7 +- src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs | 110 +++++++++++++----- 3 files changed, 85 insertions(+), 34 deletions(-) diff --git a/src/RSML.Language.Lexing/GlobalSuppressions.cs b/src/RSML.Language.Lexing/GlobalSuppressions.cs index 382fe03..15e8ecd 100644 --- a/src/RSML.Language.Lexing/GlobalSuppressions.cs +++ b/src/RSML.Language.Lexing/GlobalSuppressions.cs @@ -5,4 +5,4 @@ using System.Diagnostics.CodeAnalysis; -[assembly: SuppressMessage("Style", "IDE0046:Convert to conditional expression", Justification = "Would make the code ternary hell.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Language.Lexing.Utf8Lexer.GetNextToken(System.Buffers.SequenceReader{System.Byte}@)~OceanApocalypse.RSML.Abstractions.Diagnostics.Result{OceanApocalypse.RSML.Language.Lexing.Tokens.Token}")] +[assembly: SuppressMessage("Style", "IDE0046:Convert to conditional expression", Justification = "Would make the code ternary hell.", Scope = "member", Target = "~M:OceanApocalypse.RSML.Language.Lexing.Utf8.Utf8Lexer.GetNextToken(System.Buffers.SequenceReader{System.Byte}@,OceanApocalypse.RSML.Abstractions.Diagnostics.AbsolutePosition@)~OceanApocalypse.RSML.Abstractions.Diagnostics.Result{OceanApocalypse.RSML.Language.Lexing.Tokens.Token}")] diff --git a/src/RSML.Language.Lexing/Tokens/TokenKind.cs b/src/RSML.Language.Lexing/Tokens/TokenKind.cs index 9eef629..e43962f 100644 --- a/src/RSML.Language.Lexing/Tokens/TokenKind.cs +++ b/src/RSML.Language.Lexing/Tokens/TokenKind.cs @@ -222,5 +222,10 @@ public enum TokenKind /// /// The at symbol (@). Reserved for future use. /// - AtSymbol + AtSymbol, + + /// + /// A comment, including the # symbol. + /// + Comment } diff --git a/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs b/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs index 09862b0..4c79613 100644 --- a/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs +++ b/src/RSML.Language.Lexing/Utf8/Utf8Lexer.cs @@ -47,18 +47,17 @@ public Result GetNextToken(ref SequenceReader reader, ref AbsoluteP wasUsed = true; AbsolutePosition.ThrowIfInvalid(currentPosition); - var startLoc = reader.Consumed; - SkipWhitespaceAndComments(ref reader); + SkipWhitespaceAndComments(ref reader, ref currentPosition); if (reader.End || !reader.TryPeek(out byte b)) - return Result.Success(new Token(TokenKind.Eof, startLoc, startLoc)); + return Result.Success(new Token(TokenKind.Eof, reader.Consumed, reader.Consumed)); if (!b.IsAscii()) { return Result.Failure(new( LexerErrorCodes.InvalidData, - startLoc, currentPosition, - startLoc, currentPosition, + reader.Consumed, currentPosition, + reader.Consumed, currentPosition, "Expected an ASCII character but received a non-ASCII character.", Severity.Error )); @@ -68,33 +67,35 @@ public Result GetNextToken(ref SequenceReader reader, ref AbsoluteP // strings if (c == '"') - return ScanStringLiteral(ref reader, startLoc, ref currentPosition); + return ScanStringLiteral(ref reader, ref currentPosition); // number literals if (b.IsAsciiDigit()) - return ScanNumber(ref reader, startChar: b, startLoc, ref currentPosition); + return ScanNumber(ref reader, startChar: b, ref currentPosition); // identifiers and keywords if (b.IsAsciiLetter() || c == '_') - return ScanIdentifierOrKeyword(ref reader, startLoc, ref currentPosition); + return ScanIdentifierOrKeyword(ref reader, ref currentPosition); // standard library identifiers if (c == '$') - return ScanStdIdentifier(ref reader, startLoc, ref currentPosition); + return ScanStdIdentifier(ref reader, ref currentPosition); // member access notation if (c == '.') { reader.Advance(1); currentPosition.Column++; - return Result.Success(new Token(TokenKind.MemberAccess, startLoc, 1)); + return Result.Success(new Token(TokenKind.MemberAccess, reader.Consumed, 1)); } // punctuation if (b.IsRsmlPunctuation()) - return ScanPunctuation(ref reader, startLoc, ref currentPosition); + return ScanPunctuation(ref reader, ref currentPosition); - // todo: check for comments if Configuration.EmitComments is enabled + // comments + if (Configuration.EmitComments && b == (byte)'#') + return ScanComment(ref reader, ref currentPosition); return Result.Failure(new( code: LexerErrorCodes.FailedToLexToken, @@ -112,6 +113,7 @@ public IEnumerable Lex(ReadOnlySequence data) var reader = new SequenceReader(data); var position = AbsolutePosition.Default; + var writer = new ArrayBufferWriter((int)(data.Length / 2)); if (!position.IsValid) { @@ -121,20 +123,33 @@ public IEnumerable Lex(ReadOnlySequence data) while (Configuration.MaximumAllowedFailuresPerComponent <= 0 || failedRuns < Configuration.MaximumAllowedFailuresPerComponent) { - var token = GetNextToken(ref reader, ref position); + Span tokens = writer.GetSpan(64); + int idx = 0; - if (token.IsError) + while (idx < tokens.Length && Configuration.MaximumAllowedFailuresPerComponent <= 0 || failedRuns < Configuration.MaximumAllowedFailuresPerComponent) { - Diagnostics.Add(token.Error); - failedRuns++; - continue; + var token = GetNextToken(ref reader, ref position); + + if (token.IsError) + { + Diagnostics.Add(token.Error); + failedRuns++; + continue; + } + + if (token.Value.Kind == TokenKind.Eof) + { + return writer.WrittenSpan.ToArray(); + } + + else + { + tokens[idx] = token.Value; + idx++; + } } - if (token.Value.Kind == TokenKind.Eof) - yield break; - - else - yield return token.Value; + writer.Advance(idx); } throw new ExceededMaxAmountOfFailuresException( @@ -142,9 +157,10 @@ public IEnumerable Lex(ReadOnlySequence data) ); } - private static Result ScanNumber(ref SequenceReader reader, byte startChar, long startLoc, ref AbsolutePosition position) + private static Result ScanNumber(ref SequenceReader reader, byte startChar, ref AbsolutePosition position) { var startPos = position; + var startLoc = reader.Consumed; byte b = startChar; bool hasDotSeparator = false; @@ -173,9 +189,10 @@ private static Result ScanNumber(ref SequenceReader reader, byte st return Result.Success(new Token(TokenKind.NumericLiteral, startLoc, reader.Consumed - startLoc)); } - private static Result ScanStringLiteral(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) + private static Result ScanStringLiteral(ref SequenceReader reader, ref AbsolutePosition position) { var startPos = position; + var startLoc = reader.Consumed; bool escaping = false; if (reader.End) @@ -220,9 +237,10 @@ private static Result ScanStringLiteral(ref SequenceReader reader, return Result.Success(new Token(TokenKind.StringLiteral, startLoc, reader.Consumed - startLoc)); } - private static Result ScanStdIdentifier(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) + private static Result ScanStdIdentifier(ref SequenceReader reader, ref AbsolutePosition position) { - AbsolutePosition startPos = position; + var startPos = position; + var startLoc = reader.Consumed; do { @@ -242,9 +260,10 @@ private static Result ScanStdIdentifier(ref SequenceReader reader, : Result.Success(new Token(TokenKind.StandardLibraryIdentifier, startLoc, reader.Consumed - startLoc)); } - private static Result ScanIdentifierOrKeyword(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) + private static Result ScanIdentifierOrKeyword(ref SequenceReader reader, ref AbsolutePosition position) { var startPos = position; + var startLoc = reader.Consumed; while (!reader.End && reader.TryPeek(out byte b) && (b.IsAsciiLetter() || b.IsAsciiDigit() || b == '_')) { @@ -276,9 +295,10 @@ private static Result ScanIdentifierOrKeyword(ref SequenceReader re } } - private static Result ScanPunctuation(ref SequenceReader reader, long startLoc, ref AbsolutePosition position) + private static Result ScanPunctuation(ref SequenceReader reader, ref AbsolutePosition position) { var startPos = position; + var startLoc = reader.Consumed; reader.TryRead(out byte first); // will always work and will always be ASCII position.Column++; @@ -308,18 +328,44 @@ private static Result ScanPunctuation(ref SequenceReader reader, lo : Result.Success(new Token(kind, startLoc, reader.Consumed - startLoc)); } - private void SkipWhitespaceAndComments(ref SequenceReader reader) + private static Result ScanComment(ref SequenceReader reader, ref AbsolutePosition position) + { + var startLoc = reader.Consumed; + var found = reader.TryAdvanceToAny([(byte)'\r', (byte)'\n'], advancePastDelimiter: false); + // not advancing past cuz it gets handled in next GetNextToken call + + if (!found) // consume everything - we're EOF + reader.AdvanceToEnd(); + + int tokenLength = (int)(reader.Consumed - startLoc); + position.Column += tokenLength; + return Result.Success(new Token(TokenKind.Comment, startLoc, tokenLength)); + } + + private void SkipWhitespaceAndComments(ref SequenceReader reader, ref AbsolutePosition position) { while (!reader.End && reader.TryPeek(out byte b)) { - if (b.IsAsciiWhitespace()) + if (b.IsAsciiNewline()) + { reader.Advance(1); + position.MoveToStartOfNextLine(); + if (b == (byte)'\r' && reader.TryPeek(out byte next) && next == (byte)'\n') + reader.Advance(1); + } + else if (b.IsAsciiWhitespace()) + { + position.Column += (int)reader.AdvancePastAny([9, 11, 12, 30]); + } else if (b == (byte)'#' && !Configuration.EmitComments) - reader.TryAdvanceToAny([(byte)'\r', (byte)'\n'], advancePastDelimiter: true); - + { + _ = ScanComment(ref reader, ref position); + } else + { break; + } } }