Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/eml/eml.ml
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,8 @@ struct

let rec scan_whitespace stream columns =
match Stream.peek stream with
| Some ' ' ->
Buffer.add_char lookahead_buffer ' ';
| Some ((' ' | '\t') as c) ->
Buffer.add_char lookahead_buffer c;
Stream.junk stream;
scan_whitespace stream (columns + 1)
| _ ->
Expand Down
32 changes: 29 additions & 3 deletions test/expect/eml/tokens.ml
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,37 @@



let show input =
let scan input =
Eml.Location.reset ();

let underlying = Stream.of_string input in
let input_stream = Eml.Location.stream (fun () ->
try Some (Stream.next underlying)
with _ -> None) in

Eml.Tokenizer.scan input_stream

let show_with format input =
try
input_stream
|> Eml.Tokenizer.scan
input
|> scan
|> List.map Eml.Token.show
|> List.map format
|> List.iter print_endline
with Failure message ->
print_endline message

let show input =
show_with (fun text -> text) input

let escape_tabs text =
text
|> String.split_on_char '\t'
|> String.concat "\\t"

let show_escaped_tabs input =
show_with escape_tabs input

let%expect_test _ =
show "";
[%expect {| (1, 0) Code_block |}]
Expand Down Expand Up @@ -91,6 +106,17 @@ let%expect_test _ =
Newline
Text {| </html>|} |xxx}]

let%expect_test _ =
show_escaped_tabs "let foo =\n\t<html>\n\t</html>";
[%expect {xxx|
(1, 0) Code_block
let foo =

Options , 1
Text {|\t<html>|}
Newline
Text {|\t</html>|} |xxx}]

let%expect_test _ =
show "let foo =\n <html>\n plain";
[%expect {xxx|
Expand Down
Loading