From 4662c5879e663280f22b6c533a75315fe8b8c50b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 26 May 2026 23:22:54 -0700 Subject: [PATCH] Remove `Reject` from Hocc parser API Rather than encoding a syntax error as a `Reject` status, return an `Error` result to indicate rejection. This has the advantage of leaving the parser in its original state upon syntax error, which simplifies diagnosis/repair code paths, e.g. the `expect` function. --- bootstrap/bin/hocc/Parse.hmh | 33 ++-- bootstrap/bin/hocc/Parse.ml | 95 +++++----- bootstrap/bin/hocc/code.ml | 171 +++++++++--------- bootstrap/test/hocc/Example.expected.hm | 57 +++--- bootstrap/test/hocc/Example.expected.hmi | 26 +-- bootstrap/test/hocc/Example_b.expected.hm | 57 +++--- bootstrap/test/hocc/Example_b.expected.hmi | 26 +-- bootstrap/test/hocc/Example_c.expected.hm | 57 +++--- bootstrap/test/hocc/Example_c.expected.hmi | 26 +-- .../test/hocc/Example_introspect.expected.ml | 62 +++---- .../test/hocc/Example_introspect.expected.mli | 26 +-- bootstrap/test/hocc/Example_ml.expected.ml | 79 ++++---- bootstrap/test/hocc/Example_ml.expected.mli | 26 +-- bootstrap/test/hocc/Example_ml.hmh | 17 +- doc/tools/hocc.md | 28 +-- 15 files changed, 386 insertions(+), 400 deletions(-) diff --git a/bootstrap/bin/hocc/Parse.hmh b/bootstrap/bin/hocc/Parse.hmh index ef825ea01..a768cb42f 100644 --- a/bootstrap/bin/hocc/Parse.hmh +++ b/bootstrap/bin/hocc/Parse.hmh @@ -756,12 +756,8 @@ let hmhi scanner = | [] -> errs | _ -> List.fold ~init:errs ~f:(fun errs mal -> Error.init_mal mal :: errs) mals in - let {status; _} as parser = next token parser in - match status, errs with - | Prefix, _ -> inner scanner errs parser - | Accept (Hmhi hmhi), [] -> scanner, Ok hmhi - | Accept (Hmhi _), _ -> scanner, Error errs - | Reject _, _ -> + match next token parser with + | Error _ -> begin let msg = String.Fmt.empty |> Fmt.fmt "Unexpected token not in " @@ -770,7 +766,14 @@ let hmhi scanner = in let errs = Error.init_token scan_token msg :: errs in scanner, Error errs - | _ -> not_reached () + end + | Ok ({status; _} as parser) -> begin + match status, errs with + | Prefix, _ -> inner scanner errs parser + | Accept (Hmhi hmhi), [] -> scanner, Ok hmhi + | Accept (Hmhi _), _ -> scanner, Error errs + | _ -> not_reached () + end end in let parser = Start.Hmhi.boi in inner scanner [] parser @@ -782,12 +785,8 @@ let hmh scanner = | [] -> errs | _ -> List.fold ~init:errs ~f:(fun errs mal -> Error.init_mal mal :: errs) mals in - let {status; _} as parser = next token parser in - match status, errs with - | Prefix, _ -> inner scanner errs parser - | Accept (Hmh hmh), [] -> scanner, Ok hmh - | Accept (Hmh _), _ -> scanner, Error errs - | Reject _, _ -> begin + match next token parser with + | Error _ -> begin let msg = String.Fmt.empty |> Fmt.fmt "Unexpected token not in " @@ -797,7 +796,13 @@ let hmh scanner = let errs = Error.init_token scan_token msg :: errs in scanner, Error errs end - | _ -> not_reached () + | Ok ({status; _} as parser) -> begin + match status, errs with + | Prefix, _ -> inner scanner errs parser + | Accept (Hmh hmh), [] -> scanner, Ok hmh + | Accept (Hmh _), _ -> scanner, Error errs + | _ -> not_reached () + end end in let parser = Start.Hmh.boi in inner scanner [] parser diff --git a/bootstrap/bin/hocc/Parse.ml b/bootstrap/bin/hocc/Parse.ml index 574ba57c9..8d4c16a47 100644 --- a/bootstrap/bin/hocc/Parse.ml +++ b/bootstrap/bin/hocc/Parse.ml @@ -15116,7 +15116,6 @@ Hmhi {prelude; hocc_; postlude; eoi} | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index = function | ShiftPrefix _ -> 0L @@ -15124,7 +15123,6 @@ Hmhi {prelude; hocc_; postlude; eoi} | Reduce _ -> 2L | Prefix -> 3L | Accept _ -> 4L - | Reject _ -> 5L let hash_fold t state = state @@ -15138,7 +15136,6 @@ Hmhi {prelude; hocc_; postlude; eoi} hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token ) let cmp t0 t1 = @@ -15163,7 +15160,6 @@ Hmhi {prelude; hocc_; postlude; eoi} end | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () end | Gt -> Gt @@ -15192,7 +15188,6 @@ Hmhi {prelude; hocc_; postlude; eoi} end | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept " |> Nonterm.pp nonterm - | Reject token -> formatter |> Fmt.fmt "Reject " |> Token.pp token ) end include T @@ -15225,21 +15220,21 @@ Hmhi {prelude; hocc_; postlude; eoi} end end - let feed token = function + let feed token t = match t with | {stack={state; _} :: _; status=Prefix} as t -> begin let token_index = Token.index token in let Spec.State.{actions; _} = Array.get state Spec.states in - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> + Ok {t with status=Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> + Ok {t with status=Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> begin let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods in let reduction = Stack.Reduction.init callback_index in - Status.Reduce (token, reduction) + Ok {t with status=Status.Reduce (token, reduction)} end - | None -> Status.Reject token - in - {t with status} + | None -> Error (token, t) end | _ -> not_reached () @@ -15247,7 +15242,7 @@ Hmhi {prelude; hocc_; postlude; eoi} let open Status in match status with | ShiftPrefix (token, state) -> - {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} + Ok {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} | ShiftAccept (token, state) -> begin (* Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. *) let stack = Stack.shift ~symbol:(Token token) ~state stack in @@ -15261,47 +15256,48 @@ Hmhi {prelude; hocc_; postlude; eoi} match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} end | _ -> not_reached () end | Reduce (token, reduction) -> begin feed token {stack=Stack.reduce ~reduction stack; status=Prefix} end - | _ -> not_reached () + | _ -> halt "`step` only supports {`ShiftPrefix`, `ShiftAccept`, `Reduce`} status" - (* val walk: t -> t *) + (* val walk: t -> (t, Token.t * t) result *) let rec walk ({status; _} as t) = let open Status in match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> begin + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' + end | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t let next token ({status; _} as t) = let open Status in match status with | Prefix -> begin - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' + match feed token t with + | (Error _) as error -> error + | Ok t' -> begin + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + end end - | _ -> halt "`token` only supports `Prefix` status" + | _ -> halt "`next` only supports `Prefix` status" - let expect ({status; _} as t) = - let open Status in - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" - in + let expect t = Array.fold ~init:(Ordset.empty (module Token)) ~f:(fun expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos end #678 "./Parse.hmh" @@ -15386,12 +15382,8 @@ let hmhi scanner = | [] -> errs | _ -> List.fold ~init:errs ~f:(fun errs mal -> Error.init_mal mal :: errs) mals in - let {status; _} as parser = next token parser in - match status, errs with - | Prefix, _ -> inner scanner errs parser - | Accept (Hmhi hmhi), [] -> scanner, Ok hmhi - | Accept (Hmhi _), _ -> scanner, Error errs - | Reject _, _ -> + match next token parser with + | Error _ -> begin let msg = String.Fmt.empty |> Fmt.fmt "Unexpected token not in " @@ -15400,7 +15392,14 @@ let hmhi scanner = in let errs = Error.init_token scan_token msg :: errs in scanner, Error errs - | _ -> not_reached () + end + | Ok ({status; _} as parser) -> begin + match status, errs with + | Prefix, _ -> inner scanner errs parser + | Accept (Hmhi hmhi), [] -> scanner, Ok hmhi + | Accept (Hmhi _), _ -> scanner, Error errs + | _ -> not_reached () + end end in let parser = Start.Hmhi.boi in inner scanner [] parser @@ -15412,12 +15411,8 @@ let hmh scanner = | [] -> errs | _ -> List.fold ~init:errs ~f:(fun errs mal -> Error.init_mal mal :: errs) mals in - let {status; _} as parser = next token parser in - match status, errs with - | Prefix, _ -> inner scanner errs parser - | Accept (Hmh hmh), [] -> scanner, Ok hmh - | Accept (Hmh _), _ -> scanner, Error errs - | Reject _, _ -> begin + match next token parser with + | Error _ -> begin let msg = String.Fmt.empty |> Fmt.fmt "Unexpected token not in " @@ -15427,7 +15422,13 @@ let hmh scanner = let errs = Error.init_token scan_token msg :: errs in scanner, Error errs end - | _ -> not_reached () + | Ok ({status; _} as parser) -> begin + match status, errs with + | Prefix, _ -> inner scanner errs parser + | Accept (Hmh hmh), [] -> scanner, Ok hmh + | Accept (Hmh _), _ -> scanner, Error errs + | _ -> not_reached () + end end in let parser = Start.Hmh.boi in inner scanner [] parser diff --git a/bootstrap/bin/hocc/code.ml b/bootstrap/bin/hocc/code.ml index 188a1df5c..28d11db1f 100644 --- a/bootstrap/bin/hocc/code.ml +++ b/bootstrap/bin/hocc/code.ml @@ -324,10 +324,9 @@ let hmi_template = {|{ | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - # Common variants. + # Common variants produced by `feed`/`step`/`next`. | Prefix # Valid parse prefix; more input needed. | Accept of Nonterm.t # Successful parse result. - | Reject of Token.t # Syntax error due to unexpected token. include IdentifiableIntf.S with type t := t } @@ -341,21 +340,24 @@ let hmi_template = {|{ «starts» } - feed: Token.t -> t -> t - [@@doc "`feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, - `Reduce`, `Reject`}. `t.status` must be `Prefix`."] + feed: Token.t -> t -> result t (Token.t * t) + [@@doc "`feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting unexpected + token. `t.status` must be `Prefix`."] - step: t -> t - [@@doc "`step t` returns the result of applying one state transition to `t`. `t.status` must - be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] + step: t -> result t (Token.t * t) + [@@doc "`step t` applies one state transition to `t` and returns an `Ok t'` result or an + `Error (token, t)` rejection result due to unexpected token. `t.status` must be in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] - next: Token.t -> t -> t - [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return a - result with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`."] + next: Token.t -> t -> result t (Token.t * t) + [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return an + `Ok t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`."] expect: t -> Ordset.t Token.t Token.cmper_witness [@@doc "`expect t` returns the set of token (proto)types which `next token t` would not - reject, assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}."] + reject. `t.status` must be `Prefix`."] }|} let expand_hmi_tokens symbols ~indentation formatter = @@ -1150,7 +1152,6 @@ let hm_template = {|{ | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index t = match t with | ShiftPrefix _ -> 0 @@ -1158,7 +1159,6 @@ let hm_template = {|{ | Reduce _ -> 2 | Prefix -> 3 | Accept _ -> 4 - | Reject _ -> 5 let hash_fold t state = state @@ -1172,7 +1172,6 @@ let hm_template = {|{ hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token let cmp t0 t1 = let open Cmp @@ -1193,7 +1192,6 @@ let hm_template = {|{ | Gt -> Gt | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () | Gt -> Gt @@ -1215,7 +1213,6 @@ let hm_template = {|{ ^)(^reduction^))" | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept %f(^Nonterm.pp^)(^nonterm^)" - | Reject token -> formatter |> Fmt.fmt "Reject %f(^Token.pp^)(^token^)" } include T include Identifiable.Make(T) @@ -1234,21 +1231,21 @@ let hm_template = {|{ | {stack={state; _} :: _; status=Prefix} as t -> let token_index = Token.index token let Spec.State.{actions; _} = Array.get state Spec.states - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> Ok {t with Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> Ok {t with Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods let reduction = Stack.Reduction.init callback_index - Status.Reduce (token, reduction) - | None -> Status.Reject token - {t with status} + Ok {t with Status.Reduce (token, reduction)} + | None -> Error (token, t) | _ -> not_reached () step {stack; status} = let open Status match status with - | ShiftPrefix (token, state) -> {stack=shift token state stack; status=Prefix} + | ShiftPrefix (token, state) -> + Ok {stack=shift token state stack; status=Prefix} | ShiftAccept (token, state) -> # Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. let stack = shift token state stack @@ -1262,42 +1259,42 @@ let hm_template = {|{ match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} | _ -> not_reached () | Reduce (token, reduction) -> feed token {stack=Stack.reduce ~reduction stack; status=Prefix} | _ -> not_reached () - # walk: t -> t + # walk: t -> result t (Token.t * t) rec walk ({status; _} as t) = let open Status match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t next token ({status; _} as t) = let open Status match status with - | Status.Prefix -> - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' - | _ -> halt "`token` only supports `Prefix` status" - - expect ({status; _} as t) = - let open Status - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" + | Prefix -> + match feed token t with + | (Error _) as error -> error + | Ok t' -> + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + | _ -> halt "`next` only supports `Prefix` status" + + expect t = Array.fold ~init:(Ordset.empty Token) ~f:(fn expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos }|} @@ -2215,10 +2212,9 @@ let mli_template = {|sig | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - (* Common variants. *) + (* Common variants produced by `feed`/`step`/`next`. *) | Prefix (** Valid parse prefix; more input needed. *) | Accept of Nonterm.t (** Successful parse result. *) - | Reject of Token.t (** Syntax error due to unexpected token. *) include IdentifiableIntf.S with type t := t end @@ -2232,21 +2228,24 @@ let mli_template = {|sig «starts» end - val feed: Token.t -> t -> t - (** `feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, `Reduce`, - `Reject`}. `t.status` must be `Prefix`. *) + val feed: Token.t -> t -> (t, Token.t * t) result + (** `feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting + unexpected token. `t.status` must be `Prefix`. *) - val step: t -> t - (** `step t` returns the result of applying one state transition to `t`. `t.status` must be in + val step: t -> (t, Token.t * t) result + (** `step t` applies one state transition to `t` and returns an `Ok t'` result or an `Error + (token, t)` rejection result due to unexpected token. `t.status` must be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}. *) - val next: Token.t -> t -> t - (** `next token t` calls `feed token t` and fast-forwards via `step` calls to return a result - with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`. *) + val next: Token.t -> t -> (t, Token.t * t) result + (** `next token t` calls `feed token t` and fast-forwards via `step` calls to return an `Ok + t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`. *) val expect: t -> (Token.t, Token.cmper_witness) Ordset.t - (** `expect t` returns the set of token (proto)types which `next token t` would not reject, - assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}. *) + (** `expect t` returns the set of token (proto)types which `next token t` would not reject. + `t.status` must be `Prefix`. *) end|} let expand_mli_tokens symbols ~indentation formatter = @@ -3067,7 +3066,6 @@ let ml_template = {|struct | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index = function | ShiftPrefix _ -> 0L @@ -3075,7 +3073,6 @@ let ml_template = {|struct | Reduce _ -> 2L | Prefix -> 3L | Accept _ -> 4L - | Reject _ -> 5L let hash_fold t state = state @@ -3089,7 +3086,6 @@ let ml_template = {|struct hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token ) let cmp t0 t1 = @@ -3114,7 +3110,6 @@ let ml_template = {|struct end | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () end | Gt -> Gt @@ -3143,7 +3138,6 @@ let ml_template = {|struct end | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept " |> Nonterm.pp nonterm - | Reject token -> formatter |> Fmt.fmt "Reject " |> Token.pp token ) end include T @@ -3159,21 +3153,21 @@ let ml_template = {|struct «starts» end - let feed token = function + let feed token t = match t with | {stack={state; _} :: _; status=Prefix} as t -> begin let token_index = Token.index token in let Spec.State.{actions; _} = Array.get state Spec.states in - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> + Ok {t with status=Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> + Ok {t with status=Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> begin let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods in let reduction = Stack.Reduction.init callback_index in - Status.Reduce (token, reduction) + Ok {t with status=Status.Reduce (token, reduction)} end - | None -> Status.Reject token - in - {t with status} + | None -> Error (token, t) end | _ -> not_reached () @@ -3181,7 +3175,7 @@ let ml_template = {|struct let open Status in match status with | ShiftPrefix (token, state) -> - {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} + Ok {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} | ShiftAccept (token, state) -> begin (* Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. *) let stack = Stack.shift ~symbol:(Token token) ~state stack in @@ -3195,47 +3189,48 @@ let ml_template = {|struct match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} end | _ -> not_reached () end | Reduce (token, reduction) -> begin feed token {stack=Stack.reduce ~reduction stack; status=Prefix} end - | _ -> not_reached () + | _ -> halt "`step` only supports {`ShiftPrefix`, `ShiftAccept`, `Reduce`} status" - (* val walk: t -> t *) + (* val walk: t -> (t, Token.t * t) result *) let rec walk ({status; _} as t) = let open Status in match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> begin + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' + end | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t let next token ({status; _} as t) = let open Status in match status with | Prefix -> begin - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' + match feed token t with + | (Error _) as error -> error + | Ok t' -> begin + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + end end - | _ -> halt "`token` only supports `Prefix` status" + | _ -> halt "`next` only supports `Prefix` status" - let expect ({status; _} as t) = - let open Status in - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" - in + let expect t = Array.fold ~init:(Ordset.empty (module Token)) ~f:(fun expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos end|} diff --git a/bootstrap/test/hocc/Example.expected.hm b/bootstrap/test/hocc/Example.expected.hm index a0c306ab1..164fc6c4e 100644 --- a/bootstrap/test/hocc/Example.expected.hm +++ b/bootstrap/test/hocc/Example.expected.hm @@ -1225,7 +1225,6 @@ include [:]{ | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index t = match t with | ShiftPrefix _ -> 0 @@ -1233,7 +1232,6 @@ include [:]{ | Reduce _ -> 2 | Prefix -> 3 | Accept _ -> 4 - | Reject _ -> 5 let hash_fold t state = state @@ -1247,7 +1245,6 @@ include [:]{ hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token let cmp t0 t1 = let open Cmp @@ -1268,7 +1265,6 @@ include [:]{ | Gt -> Gt | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () | Gt -> Gt @@ -1290,7 +1286,6 @@ include [:]{ ^)(^reduction^))" | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept %f(^Nonterm.pp^)(^nonterm^)" - | Reject token -> formatter |> Fmt.fmt "Reject %f(^Token.pp^)(^token^)" } include T include Identifiable.Make(T) @@ -1317,21 +1312,21 @@ include [:]{ | {stack={state; _} :: _; status=Prefix} as t -> let token_index = Token.index token let Spec.State.{actions; _} = Array.get state Spec.states - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> Ok {t with Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> Ok {t with Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods let reduction = Stack.Reduction.init callback_index - Status.Reduce (token, reduction) - | None -> Status.Reject token - {t with status} + Ok {t with Status.Reduce (token, reduction)} + | None -> Error (token, t) | _ -> not_reached () step {stack; status} = let open Status match status with - | ShiftPrefix (token, state) -> {stack=shift token state stack; status=Prefix} + | ShiftPrefix (token, state) -> + Ok {stack=shift token state stack; status=Prefix} | ShiftAccept (token, state) -> # Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. let stack = shift token state stack @@ -1345,42 +1340,42 @@ include [:]{ match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} | _ -> not_reached () | Reduce (token, reduction) -> feed token {stack=Stack.reduce ~reduction stack; status=Prefix} | _ -> not_reached () - # walk: t -> t + # walk: t -> result t (Token.t * t) rec walk ({status; _} as t) = let open Status match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t next token ({status; _} as t) = let open Status match status with - | Status.Prefix -> - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' - | _ -> halt "`token` only supports `Prefix` status" - - expect ({status; _} as t) = - let open Status - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" + | Prefix -> + match feed token t with + | (Error _) as error -> error + | Ok t' -> + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + | _ -> halt "`next` only supports `Prefix` status" + + expect t = Array.fold ~init:(Ordset.empty Token) ~f:(fn expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos }[:"./Example.hmh":35:0+23] diff --git a/bootstrap/test/hocc/Example.expected.hmi b/bootstrap/test/hocc/Example.expected.hmi index 5612e1d1b..a5d915d26 100644 --- a/bootstrap/test/hocc/Example.expected.hmi +++ b/bootstrap/test/hocc/Example.expected.hmi @@ -239,10 +239,9 @@ include [:]{ | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - # Common variants. + # Common variants produced by `feed`/`step`/`next`. | Prefix # Valid parse prefix; more input needed. | Accept of Nonterm.t # Successful parse result. - | Reject of Token.t # Syntax error due to unexpected token. include IdentifiableIntf.S with type t := t } @@ -258,21 +257,24 @@ include [:]{ } } - feed: Token.t -> t -> t - [@@doc "`feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, - `Reduce`, `Reject`}. `t.status` must be `Prefix`."] + feed: Token.t -> t -> result t (Token.t * t) + [@@doc "`feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting unexpected + token. `t.status` must be `Prefix`."] - step: t -> t - [@@doc "`step t` returns the result of applying one state transition to `t`. `t.status` must - be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] + step: t -> result t (Token.t * t) + [@@doc "`step t` applies one state transition to `t` and returns an `Ok t'` result or an + `Error (token, t)` rejection result due to unexpected token. `t.status` must be in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] - next: Token.t -> t -> t - [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return a - result with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`."] + next: Token.t -> t -> result t (Token.t * t) + [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return an + `Ok t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`."] expect: t -> Ordset.t Token.t Token.cmper_witness [@@doc "`expect t` returns the set of token (proto)types which `next token t` would not - reject, assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}."] + reject. `t.status` must be `Prefix`."] }[:"./Example.hmhi":5:0+12] calculate: string -> zint diff --git a/bootstrap/test/hocc/Example_b.expected.hm b/bootstrap/test/hocc/Example_b.expected.hm index 583296c85..3224b2a2e 100644 --- a/bootstrap/test/hocc/Example_b.expected.hm +++ b/bootstrap/test/hocc/Example_b.expected.hm @@ -1228,7 +1228,6 @@ include | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index t = match t with | ShiftPrefix _ -> 0 @@ -1236,7 +1235,6 @@ include | Reduce _ -> 2 | Prefix -> 3 | Accept _ -> 4 - | Reject _ -> 5 let hash_fold t state = state @@ -1250,7 +1248,6 @@ include hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token let cmp t0 t1 = let open Cmp @@ -1271,7 +1268,6 @@ include | Gt -> Gt | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () | Gt -> Gt @@ -1293,7 +1289,6 @@ include ^)(^reduction^))" | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept %f(^Nonterm.pp^)(^nonterm^)" - | Reject token -> formatter |> Fmt.fmt "Reject %f(^Token.pp^)(^token^)" } include T include Identifiable.Make(T) @@ -1320,21 +1315,21 @@ include | {stack={state; _} :: _; status=Prefix} as t -> let token_index = Token.index token let Spec.State.{actions; _} = Array.get state Spec.states - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> Ok {t with Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> Ok {t with Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods let reduction = Stack.Reduction.init callback_index - Status.Reduce (token, reduction) - | None -> Status.Reject token - {t with status} + Ok {t with Status.Reduce (token, reduction)} + | None -> Error (token, t) | _ -> not_reached () step {stack; status} = let open Status match status with - | ShiftPrefix (token, state) -> {stack=shift token state stack; status=Prefix} + | ShiftPrefix (token, state) -> + Ok {stack=shift token state stack; status=Prefix} | ShiftAccept (token, state) -> # Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. let stack = shift token state stack @@ -1348,42 +1343,42 @@ include match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} | _ -> not_reached () | Reduce (token, reduction) -> feed token {stack=Stack.reduce ~reduction stack; status=Prefix} | _ -> not_reached () - # walk: t -> t + # walk: t -> result t (Token.t * t) rec walk ({status; _} as t) = let open Status match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t next token ({status; _} as t) = let open Status match status with - | Status.Prefix -> - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' - | _ -> halt "`token` only supports `Prefix` status" - - expect ({status; _} as t) = - let open Status - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" + | Prefix -> + match feed token t with + | (Error _) as error -> error + | Ok t' -> + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + | _ -> halt "`next` only supports `Prefix` status" + + expect t = Array.fold ~init:(Ordset.empty Token) ~f:(fn expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos }[:"./Example_b.hmh":38:0+23] diff --git a/bootstrap/test/hocc/Example_b.expected.hmi b/bootstrap/test/hocc/Example_b.expected.hmi index c4e8a1f4f..c33b1c1b9 100644 --- a/bootstrap/test/hocc/Example_b.expected.hmi +++ b/bootstrap/test/hocc/Example_b.expected.hmi @@ -239,10 +239,9 @@ include | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - # Common variants. + # Common variants produced by `feed`/`step`/`next`. | Prefix # Valid parse prefix; more input needed. | Accept of Nonterm.t # Successful parse result. - | Reject of Token.t # Syntax error due to unexpected token. include IdentifiableIntf.S with type t := t } @@ -258,21 +257,24 @@ include } } - feed: Token.t -> t -> t - [@@doc "`feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, - `Reduce`, `Reject`}. `t.status` must be `Prefix`."] + feed: Token.t -> t -> result t (Token.t * t) + [@@doc "`feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting unexpected + token. `t.status` must be `Prefix`."] - step: t -> t - [@@doc "`step t` returns the result of applying one state transition to `t`. `t.status` must - be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] + step: t -> result t (Token.t * t) + [@@doc "`step t` applies one state transition to `t` and returns an `Ok t'` result or an + `Error (token, t)` rejection result due to unexpected token. `t.status` must be in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] - next: Token.t -> t -> t - [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return a - result with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`."] + next: Token.t -> t -> result t (Token.t * t) + [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return an + `Ok t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`."] expect: t -> Ordset.t Token.t Token.cmper_witness [@@doc "`expect t` returns the set of token (proto)types which `next token t` would not - reject, assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}."] + reject. `t.status` must be `Prefix`."] }[:"./Example_b.hmhi":5:0+6] calculate: string -> zint diff --git a/bootstrap/test/hocc/Example_c.expected.hm b/bootstrap/test/hocc/Example_c.expected.hm index bd08429e9..7d9ebdd99 100644 --- a/bootstrap/test/hocc/Example_c.expected.hm +++ b/bootstrap/test/hocc/Example_c.expected.hm @@ -1226,7 +1226,6 @@ Parser = { | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index t = match t with | ShiftPrefix _ -> 0 @@ -1234,7 +1233,6 @@ Parser = { | Reduce _ -> 2 | Prefix -> 3 | Accept _ -> 4 - | Reject _ -> 5 let hash_fold t state = state @@ -1248,7 +1246,6 @@ Parser = { hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token let cmp t0 t1 = let open Cmp @@ -1269,7 +1266,6 @@ Parser = { | Gt -> Gt | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () | Gt -> Gt @@ -1291,7 +1287,6 @@ Parser = { ^)(^reduction^))" | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept %f(^Nonterm.pp^)(^nonterm^)" - | Reject token -> formatter |> Fmt.fmt "Reject %f(^Token.pp^)(^token^)" } include T include Identifiable.Make(T) @@ -1318,21 +1313,21 @@ Parser = { | {stack={state; _} :: _; status=Prefix} as t -> let token_index = Token.index token let Spec.State.{actions; _} = Array.get state Spec.states - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> Ok {t with Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> Ok {t with Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods let reduction = Stack.Reduction.init callback_index - Status.Reduce (token, reduction) - | None -> Status.Reject token - {t with status} + Ok {t with Status.Reduce (token, reduction)} + | None -> Error (token, t) | _ -> not_reached () step {stack; status} = let open Status match status with - | ShiftPrefix (token, state) -> {stack=shift token state stack; status=Prefix} + | ShiftPrefix (token, state) -> + Ok {stack=shift token state stack; status=Prefix} | ShiftAccept (token, state) -> # Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. let stack = shift token state stack @@ -1346,42 +1341,42 @@ Parser = { match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} | _ -> not_reached () | Reduce (token, reduction) -> feed token {stack=Stack.reduce ~reduction stack; status=Prefix} | _ -> not_reached () - # walk: t -> t + # walk: t -> result t (Token.t * t) rec walk ({status; _} as t) = let open Status match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t next token ({status; _} as t) = let open Status match status with - | Status.Prefix -> - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' - | _ -> halt "`token` only supports `Prefix` status" - - expect ({status; _} as t) = - let open Status - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" + | Prefix -> + match feed token t with + | (Error _) as error -> error + | Ok t' -> + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + | _ -> halt "`next` only supports `Prefix` status" + + expect t = Array.fold ~init:(Ordset.empty Token) ~f:(fn expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos }[:"./Example_c.hmh":36:4+23] } diff --git a/bootstrap/test/hocc/Example_c.expected.hmi b/bootstrap/test/hocc/Example_c.expected.hmi index 377432566..e5fb56da3 100644 --- a/bootstrap/test/hocc/Example_c.expected.hmi +++ b/bootstrap/test/hocc/Example_c.expected.hmi @@ -239,10 +239,9 @@ Parser = { | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - # Common variants. + # Common variants produced by `feed`/`step`/`next`. | Prefix # Valid parse prefix; more input needed. | Accept of Nonterm.t # Successful parse result. - | Reject of Token.t # Syntax error due to unexpected token. include IdentifiableIntf.S with type t := t } @@ -258,21 +257,24 @@ Parser = { } } - feed: Token.t -> t -> t - [@@doc "`feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, - `Reduce`, `Reject`}. `t.status` must be `Prefix`."] + feed: Token.t -> t -> result t (Token.t * t) + [@@doc "`feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting unexpected + token. `t.status` must be `Prefix`."] - step: t -> t - [@@doc "`step t` returns the result of applying one state transition to `t`. `t.status` must - be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] + step: t -> result t (Token.t * t) + [@@doc "`step t` applies one state transition to `t` and returns an `Ok t'` result or an + `Error (token, t)` rejection result due to unexpected token. `t.status` must be in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] - next: Token.t -> t -> t - [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return a - result with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`."] + next: Token.t -> t -> result t (Token.t * t) + [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return an + `Ok t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`."] expect: t -> Ordset.t Token.t Token.cmper_witness [@@doc "`expect t` returns the set of token (proto)types which `next token t` would not - reject, assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}."] + reject. `t.status` must be `Prefix`."] }[:"./Example_c.hmhi":5:4+12] } diff --git a/bootstrap/test/hocc/Example_introspect.expected.ml b/bootstrap/test/hocc/Example_introspect.expected.ml index 9d0daa36f..fe3e7988c 100644 --- a/bootstrap/test/hocc/Example_introspect.expected.ml +++ b/bootstrap/test/hocc/Example_introspect.expected.ml @@ -1349,7 +1349,6 @@ e | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index = function | ShiftPrefix _ -> 0L @@ -1357,7 +1356,6 @@ e | Reduce _ -> 2L | Prefix -> 3L | Accept _ -> 4L - | Reject _ -> 5L let hash_fold t state = state @@ -1371,7 +1369,6 @@ e hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token ) let cmp t0 t1 = @@ -1396,7 +1393,6 @@ e end | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () end | Gt -> Gt @@ -1425,7 +1421,6 @@ e end | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept " |> Nonterm.pp nonterm - | Reject token -> formatter |> Fmt.fmt "Reject " |> Token.pp token ) end include T @@ -1449,21 +1444,21 @@ e end end - let feed token = function + let feed token t = match t with | {stack={state; _} :: _; status=Prefix} as t -> begin let token_index = Token.index token in let Spec.State.{actions; _} = Array.get state Spec.states in - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> + Ok {t with status=Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> + Ok {t with status=Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> begin let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods in let reduction = Stack.Reduction.init callback_index in - Status.Reduce (token, reduction) + Ok {t with status=Status.Reduce (token, reduction)} end - | None -> Status.Reject token - in - {t with status} + | None -> Error (token, t) end | _ -> not_reached () @@ -1471,7 +1466,7 @@ e let open Status in match status with | ShiftPrefix (token, state) -> - {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} + Ok {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} | ShiftAccept (token, state) -> begin (* Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. *) let stack = Stack.shift ~symbol:(Token token) ~state stack in @@ -1485,47 +1480,48 @@ e match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} end | _ -> not_reached () end | Reduce (token, reduction) -> begin feed token {stack=Stack.reduce ~reduction stack; status=Prefix} end - | _ -> not_reached () + | _ -> halt "`step` only supports {`ShiftPrefix`, `ShiftAccept`, `Reduce`} status" - (* val walk: t -> t *) + (* val walk: t -> (t, Token.t * t) result *) let rec walk ({status; _} as t) = let open Status in match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> begin + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' + end | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t let next token ({status; _} as t) = let open Status in match status with | Prefix -> begin - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' + match feed token t with + | (Error _) as error -> error + | Ok t' -> begin + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + end end - | _ -> halt "`token` only supports `Prefix` status" + | _ -> halt "`next` only supports `Prefix` status" - let expect ({status; _} as t) = - let open Status in - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" - in + let expect t = Array.fold ~init:(Ordset.empty (module Token)) ~f:(fun expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos end #36 "./Example_introspect.hmh" diff --git a/bootstrap/test/hocc/Example_introspect.expected.mli b/bootstrap/test/hocc/Example_introspect.expected.mli index 617b86492..017ac540b 100644 --- a/bootstrap/test/hocc/Example_introspect.expected.mli +++ b/bootstrap/test/hocc/Example_introspect.expected.mli @@ -238,10 +238,9 @@ include sig | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - (* Common variants. *) + (* Common variants produced by `feed`/`step`/`next`. *) | Prefix (** Valid parse prefix; more input needed. *) | Accept of Nonterm.t (** Successful parse result. *) - | Reject of Token.t (** Syntax error due to unexpected token. *) include IdentifiableIntf.S with type t := t end @@ -257,19 +256,22 @@ include sig end end - val feed: Token.t -> t -> t - (** `feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, `Reduce`, - `Reject`}. `t.status` must be `Prefix`. *) + val feed: Token.t -> t -> (t, Token.t * t) result + (** `feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting + unexpected token. `t.status` must be `Prefix`. *) - val step: t -> t - (** `step t` returns the result of applying one state transition to `t`. `t.status` must be in + val step: t -> (t, Token.t * t) result + (** `step t` applies one state transition to `t` and returns an `Ok t'` result or an `Error + (token, t)` rejection result due to unexpected token. `t.status` must be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}. *) - val next: Token.t -> t -> t - (** `next token t` calls `feed token t` and fast-forwards via `step` calls to return a result - with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`. *) + val next: Token.t -> t -> (t, Token.t * t) result + (** `next token t` calls `feed token t` and fast-forwards via `step` calls to return an `Ok + t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`. *) val expect: t -> (Token.t, Token.cmper_witness) Ordset.t - (** `expect t` returns the set of token (proto)types which `next token t` would not reject, - assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}. *) + (** `expect t` returns the set of token (proto)types which `next token t` would not reject. + `t.status` must be `Prefix`. *) end \ No newline at end of file diff --git a/bootstrap/test/hocc/Example_ml.expected.ml b/bootstrap/test/hocc/Example_ml.expected.ml index 9d2ed26cd..fa72ba615 100644 --- a/bootstrap/test/hocc/Example_ml.expected.ml +++ b/bootstrap/test/hocc/Example_ml.expected.ml @@ -1350,7 +1350,6 @@ e | Reduce of Token.t * Stack.Reduction.t | Prefix | Accept of Nonterm.t - | Reject of Token.t let constructor_index = function | ShiftPrefix _ -> 0L @@ -1358,7 +1357,6 @@ e | Reduce _ -> 2L | Prefix -> 3L | Accept _ -> 4L - | Reject _ -> 5L let hash_fold t state = state @@ -1372,7 +1370,6 @@ e hash_state |> Stack.Reduction.hash_fold reduction |> Token.hash_fold token | Prefix -> hash_state | Accept nonterm -> hash_state |> Nonterm.hash_fold nonterm - | Reject token -> hash_state |> Token.hash_fold token ) let cmp t0 t1 = @@ -1397,7 +1394,6 @@ e end | Prefix, Prefix -> Eq | Accept nonterm0, Accept nonterm1 -> Nonterm.cmp nonterm0 nonterm1 - | Reject token0, Reject token1 -> Token.cmp token0 token1 | _, _ -> not_reached () end | Gt -> Gt @@ -1426,7 +1422,6 @@ e end | Prefix -> formatter |> Fmt.fmt "Prefix" | Accept nonterm -> formatter |> Fmt.fmt "Accept " |> Nonterm.pp nonterm - | Reject token -> formatter |> Fmt.fmt "Reject " |> Token.pp token ) end include T @@ -1450,21 +1445,21 @@ e end end - let feed token = function + let feed token t = match t with | {stack={state; _} :: _; status=Prefix} as t -> begin let token_index = Token.index token in let Spec.State.{actions; _} = Array.get state Spec.states in - let status = match Map.get token_index actions with - | Some (Spec.Action.ShiftPrefix state') -> Status.ShiftPrefix (token, state') - | Some (Spec.Action.ShiftAccept state') -> Status.ShiftAccept (token, state') + match Map.get token_index actions with + | Some (Spec.Action.ShiftPrefix state') -> + Ok {t with status=Status.ShiftPrefix (token, state')} + | Some (Spec.Action.ShiftAccept state') -> + Ok {t with status=Status.ShiftAccept (token, state')} | Some (Spec.Action.Reduce prod_index) -> begin let Spec.Prod.{callback=callback_index; _} = Array.get prod_index Spec.prods in let reduction = Stack.Reduction.init callback_index in - Status.Reduce (token, reduction) + Ok {t with status=Status.Reduce (token, reduction)} end - | None -> Status.Reject token - in - {t with status} + | None -> Error (token, t) end | _ -> not_reached () @@ -1472,7 +1467,7 @@ e let open Status in match status with | ShiftPrefix (token, state) -> - {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} + Ok {stack=Stack.shift ~symbol:(Token token) ~state stack; status=Prefix} | ShiftAccept (token, state) -> begin (* Shift, perform the ⊥ reduction, and extract the accepted symbol from the stack. *) let stack = Stack.shift ~symbol:(Token token) ~state stack in @@ -1486,47 +1481,48 @@ e match stack with | [] -> not_reached () | {symbol=Token _; _} :: _ -> not_reached () - | {symbol=Nonterm nonterm; _} :: _ -> {stack=[]; status=Accept nonterm} + | {symbol=Nonterm nonterm; _} :: _ -> Ok {stack=[]; status=Accept nonterm} end | _ -> not_reached () end | Reduce (token, reduction) -> begin feed token {stack=Stack.reduce ~reduction stack; status=Prefix} end - | _ -> not_reached () + | _ -> halt "`step` only supports {`ShiftPrefix`, `ShiftAccept`, `Reduce`} status" - (* val walk: t -> t *) + (* val walk: t -> (t, Token.t * t) result *) let rec walk ({status; _} as t) = let open Status in match status with | ShiftPrefix _ | ShiftAccept _ - | Reduce _ -> t |> step |> walk + | Reduce _ -> begin + match step t with + | (Error _) as error -> error + | Ok t' -> walk t' + end | Prefix - | Accept _ - | Reject _ -> t + | Accept _ -> Ok t let next token ({status; _} as t) = let open Status in match status with | Prefix -> begin - match t |> feed token |> walk with - | {status=Reject _ as status; _} -> {t with status} - | t' -> t' + match feed token t with + | (Error _) as error -> error + | Ok t' -> begin + match walk t' with + | Error _ -> Error (token, t) + | (Ok _) as ok -> ok + end end - | _ -> halt "`token` only supports `Prefix` status" + | _ -> halt "`next` only supports `Prefix` status" - let expect ({status; _} as t) = - let open Status in - let t = match status with - | Prefix -> t - | Reject _ -> {t with status=Prefix} - | _ -> halt "`expect` only supports `Prefix`/`Reject` status" - in + let expect t = Array.fold ~init:(Ordset.empty (module Token)) ~f:(fun expect token -> match next token t with - | {status=Reject _; _} -> expect - | _ -> Ordset.insert token expect + | Error _ -> expect + | Ok _ -> Ordset.insert token expect ) Token.protos end #37 "./Example_ml.hmh" @@ -1550,19 +1546,18 @@ let tokenize s = (* Calculate the result of the arithmetic expression expressed in `s`, e.g. "2 + 3 * 4". *) let calculate s = let {status; _} = List.fold_until (tokenize s) ~init:Start.Answer.boi ~f:(fun parser tok -> - let {status; _} as parser' = next tok parser in - let is_done = match status with - | Prefix -> false - | Accept _ - | Reject _ -> true - | _ -> not_reached () - in - parser', is_done + match next tok parser with + | Ok ({status; _} as parser') -> begin + match status with + | Prefix -> parser', false + | Accept _ -> parser', true + | _ -> not_reached () + end + | Error _ -> halt "Parse error" ) in match status with | Accept (Answer answer) -> answer | Prefix -> halt "Partial input" - | Reject _ -> halt "Parse error" | _ -> not_reached () let main () = diff --git a/bootstrap/test/hocc/Example_ml.expected.mli b/bootstrap/test/hocc/Example_ml.expected.mli index e21a6722f..1551b9a56 100644 --- a/bootstrap/test/hocc/Example_ml.expected.mli +++ b/bootstrap/test/hocc/Example_ml.expected.mli @@ -240,10 +240,9 @@ include sig | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - (* Common variants. *) + (* Common variants produced by `feed`/`step`/`next`. *) | Prefix (** Valid parse prefix; more input needed. *) | Accept of Nonterm.t (** Successful parse result. *) - | Reject of Token.t (** Syntax error due to unexpected token. *) include IdentifiableIntf.S with type t := t end @@ -259,21 +258,24 @@ include sig end end - val feed: Token.t -> t -> t - (** `feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, `Reduce`, - `Reject`}. `t.status` must be `Prefix`. *) + val feed: Token.t -> t -> (t, Token.t * t) result + (** `feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting + unexpected token. `t.status` must be `Prefix`. *) - val step: t -> t - (** `step t` returns the result of applying one state transition to `t`. `t.status` must be in + val step: t -> (t, Token.t * t) result + (** `step t` applies one state transition to `t` and returns an `Ok t'` result or an `Error + (token, t)` rejection result due to unexpected token. `t.status` must be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}. *) - val next: Token.t -> t -> t - (** `next token t` calls `feed token t` and fast-forwards via `step` calls to return a result - with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`. *) + val next: Token.t -> t -> (t, Token.t * t) result + (** `next token t` calls `feed token t` and fast-forwards via `step` calls to return an `Ok + t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`. *) val expect: t -> (Token.t, Token.cmper_witness) Ordset.t - (** `expect t` returns the set of token (proto)types which `next token t` would not reject, - assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}. *) + (** `expect t` returns the set of token (proto)types which `next token t` would not reject. + `t.status` must be `Prefix`. *) end #7 "./Example_ml.hmhi" diff --git a/bootstrap/test/hocc/Example_ml.hmh b/bootstrap/test/hocc/Example_ml.hmh index 21d24e88a..080f35ffc 100644 --- a/bootstrap/test/hocc/Example_ml.hmh +++ b/bootstrap/test/hocc/Example_ml.hmh @@ -54,19 +54,18 @@ let tokenize s = (* Calculate the result of the arithmetic expression expressed in `s`, e.g. "2 + 3 * 4". *) let calculate s = let {status; _} = List.fold_until (tokenize s) ~init:Start.Answer.boi ~f:(fun parser tok -> - let {status; _} as parser' = next tok parser in - let is_done = match status with - | Prefix -> false - | Accept _ - | Reject _ -> true - | _ -> not_reached () - in - parser', is_done + match next tok parser with + | Ok ({status; _} as parser') -> begin + match status with + | Prefix -> parser', false + | Accept _ -> parser', true + | _ -> not_reached () + end + | Error _ -> halt "Parse error" ) in match status with | Accept (Answer answer) -> answer | Prefix -> halt "Partial input" - | Reject _ -> halt "Parse error" | _ -> not_reached () let main () = diff --git a/doc/tools/hocc.md b/doc/tools/hocc.md index f0bb09766..977f01a93 100644 --- a/doc/tools/hocc.md +++ b/doc/tools/hocc.md @@ -844,10 +844,9 @@ parser states can be used as persistent reusable snapshots. | ShiftPrefix of Token.t * State.t | ShiftAccept of Token.t * State.t | Reduce of Token.t * Stack.Reduction.t - # Common variants. + # Common variants produced by `feed`/`step`/`next`. | Prefix # Valid parse prefix; more input needed. | Accept of Nonterm.t # Successful parse result. - | Reject of Token.t # Syntax error due to unexpected token. include IdentifiableIntf.S with type t := t } @@ -864,23 +863,24 @@ parser states can be used as persistent reusable snapshots. } } - feed: Token.t -> t -> t - [@@doc "`feed token t` returns a result with status in {`ShiftPrefix`, `ShiftAccept`, - `Reduce`, `Reject`}. `t.status` must be `Prefix`."] + feed: Token.t -> t -> result t (Token.t * t) + [@@doc "`feed token t` feeds `token` to `t` and returns an `Ok t'` result with status in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`} or an `Error (token, t)` result rejecting unexpected + token. `t.status` must be `Prefix`."] - step: t -> t - [@@doc "`step t` returns the result of applying one state transition to `t`. `t.status` must - be in {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] + step: t -> result t (Token.t * t) + [@@doc "`step t` applies one state transition to `t` and returns an `Ok t'` result or an + `Error (token, t)` rejection result due to unexpected token. `t.status` must be in + {`ShiftPrefix`, `ShiftAccept`, `Reduce`}."] - next: Token.t -> t -> t - [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return a - result with status in {`Prefix`, `Accept`, `Reject`}. `t.status` must be `Prefix`. If the - resulting status is `Reject`, the stack remains in its initial state even if one or more - reduction steps lead to the syntax error."] + next: Token.t -> t -> result t (Token.t * t) + [@@doc "`next token t` calls `feed token t` and fast-forwards via `step` calls to return an + `Ok t'` result with status in {`Prefix`, `Accept`} or an `Error (token, t)` rejection result + due to unexpected token. `t.status` must be `Prefix`."] expect: t -> Ordset.t Token.t Token.cmper_witness [@@doc "`expect t` returns the set of token (proto)types which `next token t` would not - reject, assuming status were `Prefix`. `t.status` must be in {`Prefix`, `Reject`}."] + reject. `t.status` must be `Prefix`."] } ```