Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
40e11e7
feat(unit-2): add ErrPartialEvaluation sentinel and cascading_error d…
dvhthomas Apr 6, 2026
0b877ca
feat: add error tracking to Environment for failed variable evaluations
dvhthomas Apr 6, 2026
3313fad
feat(evaluator-error-recovery): add CascadingError type and evalIdent…
dvhthomas Apr 6, 2026
6b97d84
feat(unit-3): statement-level error recovery in evaluateCalcBlockWithDoc
dvhthomas Apr 6, 2026
29c1bad
feat(unit-4): block-level error recovery in Evaluate, EvaluateBlock, …
dvhthomas Apr 6, 2026
7694dec
feat: handle ErrPartialEvaluation in doceval call sites
dvhthomas Apr 6, 2026
afa8e84
test: add golden files and integration tests for error recovery
dvhthomas Apr 6, 2026
3567c93
fix: clear stale error state on successful variable re-assignment
dvhthomas Apr 6, 2026
952fa47
fix: address code review findings for error recovery
dvhthomas Apr 6, 2026
5a10807
fix: address pre-existing issues found during review
dvhthomas Apr 6, 2026
7170d40
test: add catwalk TUI test for error recovery cascading display
dvhthomas Apr 6, 2026
82fbf07
fix: Lark site render hook supports partial evaluation results
dvhthomas Apr 6, 2026
47c2769
feat: inline per-line error display in HTML output
dvhthomas Apr 6, 2026
02c9eb6
fix: diagnostic line counter must count blank lines in HTML formatter
dvhthomas Apr 6, 2026
9557e70
fix: remove accidentally committed doceval binary
dvhthomas Apr 6, 2026
b460316
fix: display error diagnostics below source line, not inline
dvhthomas Apr 6, 2026
b015d7d
fix: restore block-level error fallback for semantic errors
dvhthomas Apr 6, 2026
3056218
fix: suppress duplicate block-level error when per-line diagnostics e…
dvhthomas Apr 6, 2026
9fe79e8
fix: collect all semantic errors in a block, not just the first
dvhthomas Apr 6, 2026
d19366b
bench: add evaluator benchmarks for valid and errored documents
dvhthomas Apr 6, 2026
0ac8d9a
fix: remove auto-bold wrapping from {{var}} interpolation
dvhthomas Apr 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,4 @@ tiny.cm

# Hugo dev server output at repo root (not site/public/)
/public/
doceval
14 changes: 8 additions & 6 deletions cmd/calcmark/cmd/embedded_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,9 @@ func TestEmbedded_BlockError_InlineAndNonZeroExit(t *testing.T) {
}

out := stdout.String()
if !strings.Contains(out, "> **CalcMark Error:**") {
t.Errorf("expected inline error blockquote, got %q", out)
// With error recovery, the error block is formatted inline (not as a blockquote)
if !strings.Contains(out, "**Error:**") && !strings.Contains(out, "undefined_var") {
t.Errorf("expected inline error marker, got %q", out)
}
if !strings.Contains(out, "# Post") {
t.Error("expected passthrough before error block")
Expand Down Expand Up @@ -310,8 +311,9 @@ func TestEmbedded_IndependentBlocks(t *testing.T) {
if !strings.Contains(out, "x = 100") {
t.Errorf("expected first block to succeed, got %q", out)
}
// Second block should error because x is not defined.
if !strings.Contains(out, "> **CalcMark Error:**") {
// Second block should error because x is not defined (each embedded block is independent).
// With error recovery, the error is shown inline (not as a blockquote).
if !strings.Contains(out, "**Error:**") && !strings.Contains(out, "undefined") {
t.Errorf("expected error for second block, got %q", out)
}
}
Expand Down Expand Up @@ -398,8 +400,8 @@ func TestEmbedded_GoldenComplexMarkdown(t *testing.T) {
`→ $870.00`,
// Scaled block (scale factor 1000, unit_categories: [Currency])
`→ $1M`,
// Error block
`> **CalcMark Error:**`,
// Error block (with error recovery, errors are formatted inline)
`**Error:** line 1: undefined_variable: undefined variable "nonexistent_var"`,
}
for _, check := range checks {
if !strings.Contains(got, check) {
Expand Down
14 changes: 12 additions & 2 deletions cmd/calcmark/cmd/eval.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
Expand Down Expand Up @@ -95,8 +96,10 @@ func runEval(args []string) error {

eval := implDoc.NewEvaluator()
eval.SetDisplayFormatter(localeFormatter())
if err := eval.Evaluate(doc); err != nil {
return returnError("evaluation error: %w", err)
evalErr := eval.Evaluate(doc)
if evalErr != nil && !errors.Is(evalErr, implDoc.ErrPartialEvaluation) {
// Fatal evaluation error (e.g., frontmatter failure) — no output
return returnError("evaluation error: %w", evalErr)
}

// Use the selected formatter for eval output (defaults to "text")
Expand All @@ -111,6 +114,13 @@ func runEval(args []string) error {
return returnError("format error: %w", err)
}

// If partial evaluation, output was formatted with partial results + diagnostics.
// Return error for non-zero exit code, but don't write JSON error envelope
// (the formatted output already contains diagnostic information).
if errors.Is(evalErr, implDoc.ErrPartialEvaluation) {
return fmt.Errorf("evaluation error: %w", evalErr)
}

return nil
}

Expand Down
96 changes: 33 additions & 63 deletions cmd/calcmark/cmd/json_error_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,6 @@ import (
"time"
)

// jsonErrorEnvelopeTest mirrors the JSON error structure we expect on stdout.
type jsonErrorEnvelopeTest struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
Line int `json:"line,omitempty"`
Code string `json:"code,omitempty"`
} `json:"error"`
}

// TestJSONError_EvalErrorProducesJSONOnStdout verifies that when --format json
// is active and an evaluation error occurs (e.g., variable redefinition),
// stdout contains a valid JSON error envelope instead of being empty.
Expand All @@ -44,28 +34,23 @@ func TestJSONError_EvalErrorProducesJSONOnStdout(t *testing.T) {
t.Fatal("expected non-zero exit code for variable redefinition")
}

// stdout must contain valid JSON
// With error recovery, stdout contains formatted JSON output (partial results
// with diagnostics), not a JSON error envelope.
if !json.Valid(stdout.Bytes()) {
t.Fatalf("stdout should be valid JSON when --format json is used, got:\nstdout: %q\nstderr: %q", stdout.String(), stderr.String())
}

// Parse and verify the error envelope
var envelope jsonErrorEnvelopeTest
if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil {
t.Fatalf("failed to unmarshal JSON error: %v\nstdout: %s", err, stdout.String())
}

if envelope.Error.Type == "" {
t.Error("error.type should not be empty")
}
if envelope.Error.Message == "" {
t.Error("error.message should not be empty")
// The output should contain the block with redefinition info
out := stdout.String()
if !strings.Contains(out, "reassign") && !strings.Contains(out, "immutable") && !strings.Contains(out, "redefinition") {
t.Errorf("JSON output should contain redefinition info, got: %s", out)
}
}

// TestJSONError_EvalErrorHasStructuredFields verifies that the JSON error
// envelope includes type, line, code, and message fields extracted from
// structured error messages.
// TestJSONError_EvalErrorHasStructuredFields verifies that with error recovery,
// redefinition errors produce formatted JSON output (with block diagnostics)
// and a non-zero exit code. The error details are in the block's diagnostics,
// not in a separate error envelope.
func TestJSONError_EvalErrorHasStructuredFields(t *testing.T) {
binary := buildCM(t)

Expand All @@ -80,29 +65,33 @@ func TestJSONError_EvalErrorHasStructuredFields(t *testing.T) {
cmd.Stdout = &stdout
cmd.Stderr = &stderr

cmd.Run() //nolint:errcheck // we expect failure
err := cmd.Run()

var envelope jsonErrorEnvelopeTest
if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil {
t.Fatalf("failed to unmarshal JSON error: %v\nstdout: %s", err, stdout.String())
// Should fail (non-zero exit) due to partial evaluation
if err == nil {
t.Error("Expected non-zero exit code for redefinition error")
}

if envelope.Error.Type != "evaluation_error" {
t.Errorf("error.type = %q, want %q", envelope.Error.Type, "evaluation_error")
}
if envelope.Error.Code != "variable_redefinition" {
t.Errorf("error.code = %q, want %q", envelope.Error.Code, "variable_redefinition")
// stderr should mention the error
if !strings.Contains(stderr.String(), "evaluation error") {
t.Errorf("stderr should mention evaluation error, got: %q", stderr.String())
}
if envelope.Error.Line == 0 {
t.Error("error.line should be non-zero for variable_redefinition")

// stdout should contain valid JSON output with block diagnostics
out := stdout.String()
if out == "" {
t.Fatal("stdout should not be empty — partial evaluation should still format output")
}
if !strings.Contains(envelope.Error.Message, "reassign") {
t.Errorf("error.message should mention reassignment, got %q", envelope.Error.Message)

// The output should contain information about the redefinition
if !strings.Contains(out, "reassign") && !strings.Contains(out, "redefinition") && !strings.Contains(out, "immutable") {
t.Errorf("JSON output should contain redefinition diagnostic info, got: %s", out)
}
}

// TestJSONError_UndefinedVariableProducesJSON verifies that undefined variable
// errors also produce JSON on stdout when --format json is active.
// errors produce formatted JSON output on stdout when --format json is active,
// and a non-zero exit code.
func TestJSONError_UndefinedVariableProducesJSON(t *testing.T) {
binary := buildCM(t)

Expand Down Expand Up @@ -130,19 +119,11 @@ func TestJSONError_UndefinedVariableProducesJSON(t *testing.T) {
if !json.Valid(stdout.Bytes()) {
t.Fatalf("stdout should be valid JSON, got:\nstdout: %q\nstderr: %q", stdout.String(), stderr.String())
}

var envelope jsonErrorEnvelopeTest
if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil {
t.Fatalf("failed to unmarshal: %v", err)
}

if envelope.Error.Type == "" {
t.Error("error.type should not be empty")
}
}

// TestJSONError_PipedStdinErrorProducesJSON verifies that the piped-stdin
// path (cm --format json, without eval subcommand) also produces JSON errors.
// path (cm --format json, without eval subcommand) produces formatted JSON
// output with diagnostics for errors, and a non-zero exit code.
func TestJSONError_PipedStdinErrorProducesJSON(t *testing.T) {
binary := buildCM(t)

Expand All @@ -166,15 +147,6 @@ func TestJSONError_PipedStdinErrorProducesJSON(t *testing.T) {
if !json.Valid(stdout.Bytes()) {
t.Fatalf("stdout should be valid JSON via piped path, got:\nstdout: %q\nstderr: %q", stdout.String(), stderr.String())
}

var envelope jsonErrorEnvelopeTest
if err := json.Unmarshal(stdout.Bytes(), &envelope); err != nil {
t.Fatalf("failed to unmarshal: %v", err)
}

if envelope.Error.Type == "" {
t.Error("error.type should not be empty")
}
}

// TestJSONError_NonJSONFormatDoesNotWriteJSON verifies that errors without
Expand All @@ -195,13 +167,11 @@ func TestJSONError_NonJSONFormatDoesNotWriteJSON(t *testing.T) {

cmd.Run() //nolint:errcheck // we expect failure

// stdout should be empty (default text format, error only on stderr)
if stdout.Len() > 0 {
t.Errorf("stdout should be empty for text format errors, got: %q", stdout.String())
}
// With error recovery, text output is still produced (partial results + diagnostics).
// stdout may contain formatted output.

// stderr should have the error
if !strings.Contains(stderr.String(), "Error") {
if !strings.Contains(stderr.String(), "Error") && !strings.Contains(stderr.String(), "error") {
t.Errorf("stderr should contain error message, got: %q", stderr.String())
}
}
Expand Down
63 changes: 63 additions & 0 deletions cmd/calcmark/tui/editor/catwalk_error_recovery_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package editor

import (
"fmt"
"io"
"strings"
"testing"

tea "charm.land/bubbletea/v2"
impldoc "github.com/CalcMark/go-calcmark/impl/document"
"github.com/CalcMark/go-calcmark/spec/document"
"github.com/cockroachdb/datadriven"
)

// TestEditorCatwalkErrorRecoveryCascading verifies TUI display of error recovery:
// root-cause errors show as non-blocked, cascading errors show as blocked (IsBlocked=true).
func TestEditorCatwalkErrorRecoveryCascading(t *testing.T) {
content := `a = 1 / 0
b = 10
c = a * 2
`

doc, err := document.NewDocument(content)
if err != nil {
t.Fatalf("Failed to create document: %v", err)
}

eval := impldoc.NewEvaluator()
_ = eval.Evaluate(doc) // ErrPartialEvaluation expected

datadriven.Walk(t, "testdata", func(t *testing.T, path string) {
if !strings.HasSuffix(path, "error_recovery_cascading") {
return
}

m := New(doc)
m.width = 80
m.height = 24
m.previewMode = PreviewFull

RunModelV2(t, path, m,
WithObserverV2("results", func(out io.Writer, m tea.Model) error {
model := m.(Model)
results := model.GetLineResults()
var buf strings.Builder
for _, r := range results {
if r.IsCalc || r.Error != "" {
errorMsg := r.Error
if errorMsg == "" {
errorMsg = `""`
} else {
errorMsg = fmt.Sprintf("%q", errorMsg)
}
buf.WriteString(fmt.Sprintf("Line %d (%s): value=%s, error=%s, blocked=%v\n",
r.LineNum, r.Source, r.Value, errorMsg, r.IsBlocked))
}
}
_, err := out.Write([]byte(buf.String()))
return err
}),
)
})
}
1 change: 1 addition & 0 deletions cmd/calcmark/tui/editor/catwalk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ z = 30`
"scale_invalid_category", // TestEditorCatwalkScaleInvalidCategory
"scale_invalid_view", // TestEditorCatwalkScaleInvalidView
"fraction_results", // TestEditorCatwalkFractionResults
"error_recovery_cascading", // TestEditorCatwalkErrorRecoveryCascading
}
for _, skip := range skipTests {
if strings.HasSuffix(path, skip) {
Expand Down
25 changes: 21 additions & 4 deletions cmd/calcmark/tui/editor/redefinition_display_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package editor

import (
"errors"
"strings"
"testing"

Expand Down Expand Up @@ -120,9 +121,25 @@ func TestRedefinitionInSingleBlock(t *testing.T) {

t.Logf("Got expected error: %v", evalErr)

// The error message should mention redefinition
if !strings.Contains(evalErr.Error(), "redefinition") &&
!strings.Contains(evalErr.Error(), "already defined") {
t.Errorf("Error should mention redefinition: %v", evalErr)
// With error recovery, Evaluate returns ErrPartialEvaluation.
// The specific redefinition error is on the block's diagnostics.
if !errors.Is(evalErr, impldoc.ErrPartialEvaluation) {
t.Errorf("Expected ErrPartialEvaluation, got: %v", evalErr)
}

// Check that the block has a redefinition error
docBlocks := doc.GetBlocks()
foundRedef := false
for _, node := range docBlocks {
if cb, ok := node.Block.(*document.CalcBlock); ok {
if cb.Error() != nil && (strings.Contains(cb.Error().Error(), "redefinition") ||
strings.Contains(cb.Error().Error(), "already defined") ||
strings.Contains(cb.Error().Error(), "reassign")) {
foundRedef = true
}
}
}
if !foundRedef {
t.Error("Expected a block to have redefinition error in diagnostics")
}
}
24 changes: 20 additions & 4 deletions cmd/calcmark/tui/editor/results.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,13 @@ func (m *Model) GetLineResults() []LineResult {
lr.Diagnostic = diag
lr.Error = diag.Code + ": " + diag.Message // Legacy string for backwards compat

// Check if this is a cascading (blocked) error
if isUndefinedVarError(lr.Error) {
// Check if this is a cascading (blocked) error.
// The evaluator marks cascading errors with diagnostic code
// "cascading_error" and hint severity. Also check the legacy
// string-match path for undefined variable errors.
if diag.Code == "cascading_error" {
lr.IsBlocked = true
} else if isUndefinedVarError(lr.Error) {
varName := extractVarFromUndefinedError(lr.Error)
if varName != "" && blockedVars[varName] {
lr.IsBlocked = true
Expand Down Expand Up @@ -158,8 +163,12 @@ func (m *Model) GetLineResults() []LineResult {
if showErrorHere {
lr.Error = blockError.Error()

// Check if this is a cascading (blocked) error
if isUndefinedVarError(lr.Error) {
// Check if this is a cascading (blocked) error.
// In the fallback path we don't have a diagnostic code,
// so check the error message for the cascading pattern.
if isCascadingErrorMessage(lr.Error) {
lr.IsBlocked = true
} else if isUndefinedVarError(lr.Error) {
varName := extractVarFromUndefinedError(lr.Error)
if varName != "" && blockedVars[varName] {
lr.IsBlocked = true
Expand Down Expand Up @@ -309,6 +318,13 @@ func isUndefinedVarError(errMsg string) bool {
strings.Contains(lowerErr, "undefined_variable")
}

// isCascadingErrorMessage checks if an error message matches the cascading error
// pattern produced by CascadingError ("depends on errored variable ...").
// Used in the blockError fallback path where no structured diagnostic code is available.
func isCascadingErrorMessage(errMsg string) bool {
return strings.Contains(errMsg, "depends on errored variable")
}

// extractVarFromUndefinedError extracts the variable name from an undefined variable error.
// Returns empty string if no variable name found.
func extractVarFromUndefinedError(errMsg string) string {
Expand Down
Loading