diff --git a/AGENTS.md b/AGENTS.md index 7a654c4a3..750519f25 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,10 +4,10 @@ Respond to the user in Japanese by default unless they explicitly request another language. ## Project Structure & Module Organization -Core runtime code lives in `doeff/`: monadic primitives in `program.py`, execution helpers in `core.py` and `interpreter.py`, and effect definitions under `effects/` with corresponding handlers in `handlers/`. Shared utilities sit beside them in `utils.py`, `types.py`, and `cache.py`. Tests that exercise each capability reside in `tests/`, while runnable samples land in `examples/`. Workspace extensions such as OpenAI, Gemini, and pinjected bridges are published from `packages/`; keep connector-specific assets inside their respective subpackages. Use `docs/` for design notes or long-form guides that support future contributors. +Core runtime code lives in `doeff/`: monadic primitives in `program.py`, the `@do` decorator in `do.py`, execution via `run()` in `run.py`, result types in `result.py`, and handler utilities in `handler_utils.py`. CLI commands are in `cli/` with auto-discovery (`discovery.py`) and execution (`run_services.py`). Effect definitions and handlers live in `packages/doeff-core-effects/` (effects in `effects.py`, handlers in `handlers.py`, scheduler in `scheduler.py`). Tests that exercise each capability reside in `tests/`, while runnable samples land in `examples/`. Workspace extensions such as OpenAI, Gemini, agents, and pinjected bridges are published from `packages/`; keep connector-specific assets inside their respective subpackages. Use `docs/` for design notes or long-form guides that support future contributors. ## Build, Test, and Development Commands -Install everything (including dev tools) with `make sync`. Use `uv run pytest` for the full suite, `uv run pytest tests/test_cache.py::test_cache_eviction` to target a single scenario, and `uv run pyright` for static typing. Build distributable artifacts via `uv run python -m build` before publishing packages. +Install everything (including dev tools) with `make sync`. Use `uv run pytest` for the full suite, `uv run pytest tests/test_core_effects.py::test_reader_ask` to target a single scenario, and `uv run pyright` for static typing. Build distributable artifacts via `uv run python -m build` before publishing packages. **WARNING — Stale Rust VM builds:** `uv sync --group dev` does NOT reliably rebuild the Rust VM extension (`packages/doeff-vm`). If you edit any `.rs` file under `packages/doeff-vm/src/`, you MUST run `make sync` (or `cd packages/doeff-vm && maturin develop --release`) to rebuild. Failing to do so will run tests against a stale binary, producing phantom failures that look like real regressions but disappear after a clean rebuild. Always use `make sync` instead of bare `uv sync`. diff --git a/README.md b/README.md index 8162850ee..b4cc71df0 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ def counter_program(): # Compose handlers explicitly by calling each handler installer. prog = counter_program() -prog = writer()(prog) +prog = writer(prog) prog = state()(prog) prog = reader(env={"greeting": "hello"})(prog) result = run(scheduled(prog)) @@ -98,14 +98,14 @@ Scheduler effects (from `doeff_core_effects.scheduler`): Built-in handlers (from `doeff_core_effects.handlers`): -| Handler | Factory | Effects handled | +| Handler | Installer | Effects handled | | --- | --- | --- | | Reader | `reader(env={...})` | `Ask` | | Lazy Ask | `lazy_ask(env={...})` | `Ask`, `Local` (with caching) | | State | `state(initial={...})` | `Get`, `Put` | -| Writer | `writer()` | `Tell` / `WriterTellEffect` | +| Writer | `writer` | `Tell` / `WriterTellEffect` | | Try | `try_handler` | `Try` | -| Slog | `slog_handler()` | `Slog` | +| Slog | `slog_handler` | `Slog` | | Local | `local_handler` | `Local` | | Listen | `listen_handler` | `Listen` | | Await | `await_handler()` | `Await` | @@ -131,7 +131,7 @@ def main(): results = yield Gather(t1, t2) return results -result = run(scheduled(writer()(main()))) +result = run(scheduled(writer(main()))) print(result) # ['a', 'b'] ``` @@ -141,22 +141,20 @@ print(result) # ['a', 'b'] # Run a program with auto-discovered interpreter doeff run --program myapp.module.program -# With explicit interpreter -doeff run --program myapp.program --interpreter myapp.interpreter - -# With environment -doeff run --program myapp.program --env myapp.default_env - -# Inline code +# Inline Python code doeff run -c 'return 42' -# Apply transform (T -> Program[U]) -doeff run --program myapp.program --apply myapp.transforms.wrap +# Inline Hy code (preferred — compose handlers inline) +doeff run --hy '(import myapp [p]) (import doeff-core-effects [lazy-ask]) ((lazy-ask :env myapp.env_dict) p)' # JSON output doeff run --program myapp.program --format json ``` +> **Note**: Legacy flags `--interpreter`, `--env`, `--set`, `--apply`, and +> `--transform` still work but emit deprecation warnings. Use `--hy` for +> inline handler composition instead. + The CLI supports automatic interpreter/environment discovery via `doeff-indexer`. See `docs/14-cli-auto-discovery.md` for marker syntax and hierarchy rules. @@ -204,10 +202,13 @@ make bench-smoke ## Development ```bash -uv sync --reinstall # rebuild Rust VM -uv run pytest +make sync # install deps + rebuild Rust VM (maturin develop --release) +uv run pytest # run full test suite ``` +> **Warning**: `uv sync` alone does NOT rebuild the Rust VM extension. Always +> use `make sync` after editing `.rs` files under `packages/doeff-vm/`. + ## License MIT License. See `LICENSE`. diff --git a/docs/01-getting-started.md b/docs/01-getting-started.md index b8e60fa5c..141bc6e55 100644 --- a/docs/01-getting-started.md +++ b/docs/01-getting-started.md @@ -34,8 +34,9 @@ uv add doeff Let's write a simple program that uses state management and logging: ```python -from doeff import do, Program, Put, Get, Tell -from doeff import run, default_handlers +from doeff import do, run, Pure +from doeff_core_effects import Get, Put, Tell +from doeff_core_effects.handlers import state, writer @do def counter_program(): @@ -52,8 +53,11 @@ def counter_program(): return final_count def main(): - result = run(counter_program(), default_handlers()) - print(f"Result: {result.value}") + prog = counter_program() + prog = writer(prog) + prog = state()(prog) + result = run(prog) + print(f"Result: {result}") main() ``` @@ -67,27 +71,32 @@ Result: 1 Let's break down what's happening: -1. **`@do` decorator**: Converts a generator function into a reusable `KleisliProgram` +1. **`@do` decorator**: Converts a generator function into a callable that produces an `Expand` program node 2. **`yield` effects**: Each `yield` *performs* an algebraic effect — the program suspends and the handler processes it 3. **Effects** (algebraic effect operations): - `Put("counter", value)` - State effect: sets state (handled by the State handler) - `Get("counter")` - State effect: retrieves state - `Tell(message)` - Writer effect: appends to log (handled by the Writer handler) -4. **`run`**: Executes programs with the provided effect handlers -5. **Result**: Returns a `RunResult` with `.value` for success or `.error` for failure +4. **Handler composition**: Handlers are composed by wrapping the program: `writer(state()(prog))` +5. **`run`**: Executes the fully-handled program and returns the raw result value -Examples in this guide use `default_handlers()` / `default_async_handlers()` only to install the -builtin runtime preset. For custom handler composition, call a Program -> Program handler installer -directly, for example `handler(program)`, instead of passing custom handlers directly to `run()` / -`async_run()`. +Handler composition is done by applying each handler to the program. +Each handler is a `Program -> Program` installer — compose them by nesting calls rather than +passing handlers to `run()`. Some handlers like `writer` and `slog_handler` are pre-installed +(used directly), while others like `reader(env=...)` and `state(initial=...)` are factories +that return an installer. ## Key Concepts ### Programs are Lazy -Programs don't execute until you call `run()` or `async_run()`: +Programs don't execute until you call `run()`: ```python +from doeff import do, run +from doeff_core_effects import Tell +from doeff_core_effects.handlers import writer + @do def my_program(): yield Tell("This won't execute yet") @@ -97,7 +106,8 @@ def my_program(): program = my_program() # Now it executes -result = run(program, default_handlers()) +prog = writer(program) +result = run(prog) ``` ### Programs are Reusable @@ -105,6 +115,10 @@ result = run(program, default_handlers()) Unlike Python generators, `@do` functions can be called multiple times: ```python +from doeff import do, run +from doeff_core_effects import Tell +from doeff_core_effects.handlers import writer + @do def format_label(label: str): yield Tell(f"Formatting {label}") @@ -115,8 +129,8 @@ prog1 = format_label("alpha") prog2 = format_label("beta") # Each execution is independent -result1 = run(prog1, default_handlers()) -result2 = run(prog2, default_handlers()) +result1 = run(writer(prog1)) +result2 = run(writer(prog2)) ``` ### Effects are Composable @@ -124,6 +138,9 @@ result2 = run(prog2, default_handlers()) You can compose effectful programs together — a key advantage of algebraic effects: ```python +from doeff import do +from doeff_core_effects import Get, Put, Tell + @do def setup(): yield Put("config", {"debug": True}) @@ -144,43 +161,47 @@ def full_program(): ## Running with Initial State -You can provide initial environment and store: +You can provide initial environment and state via handler arguments: ```python -from doeff import run, default_handlers - -# Pass initial environment and store -result = run( - my_program(), - default_handlers(), - env={"database_url": "postgresql://localhost/mydb"}, - store={"user_id": 123} -) +from doeff import do, run +from doeff_core_effects import Ask, Get +from doeff_core_effects.handlers import reader, state + +@do +def my_program(): + db_url = yield Ask("database_url") + user_id = yield Get("user_id") + return f"Connected to {db_url} as user {user_id}" + +# Pass initial environment to reader and initial state to state +prog = my_program() +prog = state(initial={"user_id": 123})(prog) +prog = reader(env={"database_url": "postgresql://localhost/mydb"})(prog) +result = run(prog) ``` ## Error Handling -`run()` and `async_run()` always return a `RunResult`: +`run()` returns the raw result value on success, or raises an exception on failure: ```python -from doeff import run, default_handlers +from doeff import do, run +from doeff_core_effects import Tell +from doeff_core_effects.handlers import writer -result = run(my_program(), default_handlers()) - -# Check success with methods (use parentheses!) -if result.is_ok(): - print(f"Success: {result.value}") -else: - print(f"Error: {result.error}") +@do +def my_program(): + yield Tell("working...") + return 42 -# Pattern matching (Python 3.10+) -from doeff import Ok, Err +prog = writer(my_program()) -match result.result: - case Ok(value): - print(f"Success: {value}") - case Err(error): - print(f"Error: {error}") +try: + result = run(prog) + print(f"Success: {result}") +except Exception as e: + print(f"Error: {e}") ``` ## Common Patterns @@ -188,6 +209,10 @@ match result.result: ### Combining Multiple Effects ```python +from doeff import do, run +from doeff_core_effects import Ask, Get, Put, Tell +from doeff_core_effects.handlers import reader, state, writer + @do def complex_workflow(): # Reader effect - get configuration @@ -199,16 +224,22 @@ def complex_workflow(): # Writer effect - log progress yield Tell("Processing data...") - # Async effect - await async operations - data = yield Await(fetch_data_async()) + connection = yield Get("connection") + return connection - # Return final result - return len(data) +prog = complex_workflow() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={"database_url": "postgresql://localhost/mydb"})(prog) +result = run(prog) ``` ### Conditional Logic ```python +from doeff import do +from doeff_core_effects import Get, Put, Tell + @do def conditional_program(): count = yield Get("count") @@ -245,15 +276,17 @@ def my_program(): Only yield `Effect` or `Program` instances: ```python +from doeff import Pure + # Wrong - yielding a plain value @do def wrong(): value = yield 42 # Error! -# Right - wrap in Program.pure() +# Right - wrap in Pure() @do def right(): - value = yield Program.pure(42) # OK + value = yield Pure(42) # OK # Or just return it return 42 ``` @@ -263,12 +296,15 @@ def right(): Don't reuse Program objects after running them with sub-effects: ```python +from doeff import run +from doeff_core_effects.handlers import writer + # If you need to run a program multiple times, call the function again prog = my_program() -result1 = run(prog, default_handlers()) +result1 = run(writer(prog)) # Don't reuse prog - create a new one prog2 = my_program() -result2 = run(prog2, default_handlers()) +result2 = run(writer(prog2)) ``` ## Next Steps @@ -285,38 +321,44 @@ Now that you understand the basics, explore: ### Common Imports ```python +# Core from doeff import ( do, # Decorator for creating programs - Program, # Program type - - # State effects - Get, Put, Modify, - - # Reader effects - Ask, Local, - - # Writer effects - Tell, Listen, - - # Async effects - Await, Gather, + run, # Execute a program, returns raw value + Pure, # Wrap a plain value as a Program + WithObserve, # Install an observer around a body program + Cancel, # Cancel a spawned task +) - # Error handling - Try, +# Effects +from doeff_core_effects import ( + Get, Put, # State effects + Ask, Local, # Reader effects + Tell, # Writer effect + slog, # Structured logging helper +) - # Result types - Ok, Err, Result, +# Handlers +from doeff_core_effects.handlers import ( + reader, # Reader handler (resolves Ask) + state, # State handler (resolves Get/Put) + writer, # Writer handler (resolves Tell) +) - # Execution functions - run, async_run, - default_handlers, default_async_handlers, +# Scheduler (for concurrency) +from doeff_core_effects.scheduler import ( + scheduled, # Wrap program for concurrent execution + Spawn, Wait, # Spawn a task and wait for it + Gather, Race, # Gather/race multiple tasks ) ``` ### Basic Program Template ```python -from doeff import do, Tell, run, default_handlers +from doeff import do, run +from doeff_core_effects import Tell +from doeff_core_effects.handlers import writer @do def my_program(): @@ -325,8 +367,9 @@ def my_program(): return "result" def main(): - result = run(my_program(), default_handlers()) - print(result.value) + prog = writer(my_program()) + result = run(prog) + print(result) if __name__ == "__main__": main() diff --git a/docs/02-core-concepts.md b/docs/02-core-concepts.md index c5b3bb585..1dd6b5e2c 100644 --- a/docs/02-core-concepts.md +++ b/docs/02-core-concepts.md @@ -10,17 +10,17 @@ This chapter defines the doeff execution model: ## Program Model `Program[T]` is the user-facing program type for effectful computations. -In the type model, `Program[T]` is `DoExpr[T]`. +In the type model, `Program[T]` is `DoExpr[T]`. It is a virtual type alias +with no methods — use the standalone constructors `Pure(x)`, `Perform(e)`, etc. ```python -class DoExpr(Generic[T]): - def map(self, f): ... - def flat_map(self, f): ... - @staticmethod - def pure(value): ... +# DoExpr is a virtual base type (isinstance check only). +# No .map(), .flat_map(), or .pure() methods exist on it. +class DoExpr(metaclass=_DoExprMeta): + """isinstance(x, DoExpr) returns True for any program node.""" -class DoCtrl(DoExpr[T]): - ... +# DoCtrl subtypes are the concrete IR nodes. +# Pure, Perform, Expand, WithHandlerType, WithObserve, ... Program = DoExpr ``` @@ -30,13 +30,14 @@ Program = DoExpr The execution boundary is explicit: - `DoExpr[T]`: control expression evaluated by VM -- `DoCtrl[T]`: concrete control nodes (`Pure`, `Call`, `Map`, `FlatMap`, `Perform`, ...) +- `DoCtrl` subtypes: concrete control nodes (`Pure`, `Expand`, `Perform`, `WithHandlerType`, `WithObserve`, ...) - `EffectValue[T]`: operation payload (`Ask`, `Get`, `Put`, `Tell`, ...) Effect values are lifted into control IR with `Perform(effect)`. ```python -from doeff import Ask, Perform +from doeff import Perform +from doeff_core_effects import Ask expr = Perform(Ask("key")) ``` @@ -73,46 +74,45 @@ yield expr ≡ Bind(expr, λresult. rest_of_program) Core control nodes include: - `Pure(value)` literal node -- `Call(f, args, kwargs, metadata)` invocation node -- `Eval(expr, handlers)` scoped evaluation node -- `Map(source, f)` composition node -- `FlatMap(source, f)` bind node +- `Expand(Apply(...))` invocation node — `@do` functions return `Expand` when called - `Perform(effect)` effect dispatch node -- handler scoping node used internally by handler installers -- continuation/control nodes such as `Resume`, `Transfer`, `Delegate`, `ResumeContinuation` +- `WithHandlerType(handler, body)` handler scoping node +- `WithObserve(observer, body)` observation/interception node +- `Apply(target, args)` function application node +- `Pass(effect, k)` re-perform to outer handler +- continuation/control nodes such as `Resume`, `Transfer`, `ResumeThrow`, `TransferThrow` These nodes are VM syntax, so composition remains in IR. ## `@do` and Macro Expansion -`@do` returns a `KleisliProgram`. Calling it emits a `Call` node. +`@do` returns a `Callable[..., Expand]`. Calling it emits an `Expand` node. -`KleisliProgram.__call__()`: +The `@do` wrapper: -1. Computes argument unwrap policy from annotations. -2. Lifts unwrapable effect arguments to `Perform(arg)`. -3. Wraps pass-through values as `Pure(arg)`. -4. Returns `Call(Pure(func), args, kwargs, metadata)`. +1. Wraps the generator function in a thunk. +2. Returns `Expand(Apply(Pure(Callable(thunk)), []))`. Example: ```python -from doeff import Ask, do +from doeff import do +from doeff_core_effects import Ask @do def fetch_user(user_id: int): db = yield Ask("db") return db[user_id] -program = fetch_user(Ask("active_user_id")) +program = fetch_user(42) ``` -`program` above is a `DoExpr` (`Call`) that is evaluated by `run`/`async_run`. +`program` above is a `DoExpr` (`Expand`) that is evaluated by `run`. ### Non-Generator `@do` Functions `@do` also supports non-generator functions that use plain `return` and never -`yield`. In that case, call-time expansion still produces a valid `Call` node, +`yield`. In that case, call-time expansion still produces a valid `Expand` node, and runtime evaluation returns the function result normally. ### Metadata Preservation @@ -122,23 +122,6 @@ The decorator preserves normal function metadata so tooling still works: the inspectable signature. This metadata preservation is part of the public behavior contract for `@do`. -### Method Decoration via `__get__` - -`KleisliProgram` implements descriptor binding (`__get__`), so `@do` works on -instance methods the same way as regular Python methods. The bound instance is -applied at call time before `Call(...)` construction. - -### Kleisli `>>` Composition - -Kleisli arrows support `>>` composition (alias of `and_then_k`) to chain -`KleisliProgram` values directly. This is Kleisli-level composition, not VM -instruction composition. - -```python -fetch_user_posts = fetch_user >> fetch_posts -program = fetch_user_posts(7) -``` - ## Rust VM Execution Pipeline Runtime flow: @@ -155,47 +138,32 @@ Key properties: - low-overhead tag-based dispatch in hot paths - effect payload fields stay opaque to VM core and are interpreted by handlers -## `run` and `async_run` +## `run` -Both entrypoints accept: +`run(doexpr)` is the single entrypoint. It accepts one argument and returns +the raw result value directly. -- `DoExpr[T]` -- raw effect values, normalized at the boundary to `Perform(effect)` +- `DoExpr[T]` — any program node +- raw effect values are normalized at the boundary to `Perform(effect)` -Conceptual sync runner: +For async/concurrent programs, wrap with `scheduled()`: ```python -def run(program, handlers): - state = init(program, handlers) - while True: - out = step(state) - if out is Done: - return value - if out is Failed: - raise error - if out is Continue: - state = out.state +from doeff import run +from doeff_core_effects.scheduler import scheduled + +result = run(scheduled(prog)) ``` -Conceptual async runner: +Conceptual runner: ```python -async def async_run(program, handlers): - state = init(program, handlers) - while True: - out = step(state) - if out is Done: - return value - if out is Failed: - raise error - if out is Continue: - state = out.state - if out is PythonAsyncSyntaxEscape: - resolved = await out.awaitable - state = out.resume(resolved) +def run(doexpr): + vm = PyVM() + return vm.run(doexpr) ``` -`PythonAsyncSyntaxEscape` is handled in the async execution path. +There is no `async_run` — use `run(scheduled(prog))` instead. ## Handler Contract @@ -204,38 +172,57 @@ Handlers interpret effects with this shape: - input: `(effect, k)` - output: `DoExpr` +Handlers use `yield Resume(k, value)` to resume the continuation with a value, +`yield Pass(effect, k)` to forward the effect to an outer handler, or +`yield effect` to re-perform an effect in the handler body. + If a host handler returns an effect value, runtime normalizes it through `Perform(effect)` before continuing. -## WithIntercept Contract +## WithObserve Contract -`WithIntercept(f, expr, types=None, mode="include")` installs scoped interception for yielded values. -The interceptor `f` receives the matched yield and must return a `DoExpr`: +`WithObserve(observer, body)` installs scoped observation for yielded values. +The observer receives matched yields and can inspect or log them. -- return the original effect/control value to pass through unchanged -- return a different `Effect` or `Program` to replace dispatch for that step -- `f` may yield effects itself; those yields skip the same interceptor layer (re-entrancy guard) +```python +from doeff import WithObserve + +def my_observer(effect, k): + print(f"observed: {effect}") + +prog = WithObserve(my_observer, body) +``` ## Composition -IR-level composition primitives: +Use standalone constructors for building program nodes: -- `Program.pure(x)` -> `Pure(x)` -- `expr.map(f)` -> `Map(expr, f)` -- `expr.flat_map(f)` -> `FlatMap(expr, f)` +- `Pure(x)` — lift a plain value into a program node +- `Perform(effect)` — lift an effect into a program node Effect payloads compose after lifting: ```python -from doeff import Ask, Perform +from doeff import Pure, Perform +from doeff_core_effects import Ask -program = Perform(Ask("key")).map(str.upper) +program = Perform(Ask("key")) +``` + +Handler composition uses the `handler(body)` pattern: + +```python +from doeff_core_effects.handlers import reader, state, writer + +prog = writer(state(initial={"count": 0})(body())) ``` ## Core Example ```python -from doeff import Ask, Get, Put, Tell, default_handlers, do, run +from doeff import do, run +from doeff_core_effects import Ask, Get, Put, Tell +from doeff_core_effects.handlers import reader, state, writer @do def update_counter(): @@ -245,21 +232,20 @@ def update_counter(): yield Tell(f"counter updated: {current} -> {current + 1}") return current + 1 -result = run( - update_counter(), - handlers=default_handlers(), - env={"counter_key": "count"}, - store={"count": 0}, -) +prog = writer(state(initial={"count": 0})(reader(env={"counter_key": "count"})(update_counter()))) +result = run(prog) ``` -Here `handlers=default_handlers()` is installing the builtin runtime preset. For custom handler -composition, prefer direct `handler(program)` calls. +Here handlers are composed individually: `writer(state()(reader()(prog)))`. +Each handler is a `Program -> Program` installer. `writer` and `slog_handler` are +pre-installed handlers (used directly), while `reader(env=...)` and `state(initial=...)` +are factories that return an installer. ## Summary -- `Program[T]` is `DoExpr[T]` +- `Program[T]` is `DoExpr[T]` — a virtual type alias with no methods - effect dispatch is explicit via `Perform(effect)` - VM evaluates control IR and handlers interpret effect payloads -- `@do` calls emit `Call` nodes that are evaluated by the runtime -- `run` and `async_run` share core stepping semantics with different async integration paths +- `@do` calls emit `Expand` nodes that are evaluated by the runtime +- `run(doexpr)` takes a single argument and returns the raw value +- for async/concurrent execution, use `run(scheduled(prog))` diff --git a/docs/03-basic-effects.md b/docs/03-basic-effects.md index 63c55c15f..54e7c8dd5 100644 --- a/docs/03-basic-effects.md +++ b/docs/03-basic-effects.md @@ -19,7 +19,10 @@ Reader effects provide read-only access to an environment/configuration that flo `Ask(key)` retrieves a value from the environment: ```python -from doeff import do, Ask, Tell, run, default_handlers +from doeff import do, run +from doeff_core_effects import Ask, Tell +from doeff_core_effects.handlers import reader, writer +from doeff_core_effects.scheduler import scheduled @do def connect_to_database(): @@ -30,15 +33,14 @@ def connect_to_database(): # Run with environment def main(): - result = run( - connect_to_database(), - default_handlers(), - env={ - "database_url": "postgresql://localhost/mydb", - "timeout": 30 - } - ) - print(result.value) # "Connected to postgresql://localhost/mydb" + prog = connect_to_database() + prog = writer(prog) + prog = reader(env={ + "database_url": "postgresql://localhost/mydb", + "timeout": 30 + })(prog) + result = run(scheduled(prog)) + print(result) # "Connected to postgresql://localhost/mydb" main() ``` @@ -51,17 +53,19 @@ main() ### Ask with Missing Keys -If a key is not in the environment, `Ask` raises `MissingEnvKeyError`: +If a key is not in the environment, `Ask` raises `KeyError`: ```python @do def requires_config(): - # Raises MissingEnvKeyError if "timeout" not in env + # Raises KeyError if "timeout" not in env timeout = yield Ask("timeout") return timeout -# Provide required keys via env parameter -result = run(requires_config(), default_handlers(), env={"timeout": 30}) +# Provide required keys via reader handler +prog = requires_config() +prog = reader(env={"timeout": 30})(prog) +result = run(scheduled(prog)) ``` ### Lazy Program Evaluation in Environment @@ -69,6 +73,8 @@ result = run(requires_config(), default_handlers(), env={"timeout": 30}) When an environment value is a `Program`, it is evaluated lazily on first access. The result is cached for subsequent `Ask` calls with the same key: ```python +from doeff_time.effects.time import Delay + @do def expensive_computation(): yield Tell("Computing config...") @@ -82,11 +88,9 @@ def use_config(): return config # Pass a Program as the env value - evaluated lazily on first Ask -result = run( - use_config(), - default_handlers(), - env={"config": expensive_computation()} # Program, not value -) +prog = use_config() +prog = reader(env={"config": expensive_computation()})(prog) +result = run(scheduled(prog)) ``` This is useful for: @@ -148,7 +152,10 @@ def fetch_data(): ### Reader Pattern Example ```python -from doeff import run, default_handlers +from doeff import do, run +from doeff_core_effects import Ask +from doeff_core_effects.handlers import reader +from doeff_core_effects.scheduler import scheduled @do def application(): @@ -163,12 +170,10 @@ def application(): # Initialize with config def main(): - result = run( - application(), - default_handlers(), - env={"config": {"option1": "value1", "option2": "value2"}} - ) - print(result.value) + prog = application() + prog = reader(env={"config": {"option1": "value1", "option2": "value2"}})(prog) + result = run(scheduled(prog)) + print(result) main() ``` @@ -194,10 +199,14 @@ def read_counter(): **Raises:** `KeyError` if key doesn't exist. Use `Try` to handle missing keys: ```python +from doeff import Ok, Err + @do def safe_read(): result = yield Try(Get("maybe_missing")) - return result.ok() if result.is_ok() else "default" + if isinstance(result, Ok): + return result.value + return "default" ``` ### Put - Write State @@ -218,37 +227,39 @@ def initialize_state(): - Overwrites existing value - Returns `None` -### Modify - Transform State +### Get + Put - Transform State -`Modify(key, func)` applies a function to current state and returns the new value: +Since `Modify` has been removed, use `Get` followed by `Put` to transform state: ```python @do def increment_counter(): - # Get, transform, and set in one operation - new_value = yield Modify("counter", lambda x: x + 1) + # Get, transform, and set + current = yield Get("counter") + new_value = current + 1 + yield Put("counter", new_value) yield Tell(f"Counter now: {new_value}") return new_value ``` **Returns:** The new (transformed) value after applying the function. -**Missing-key behavior:** If the key doesn't exist, `Modify` passes `None` to the function (unlike `Get` which raises `KeyError`). +**Missing-key behavior:** `Get` raises `KeyError` if the key doesn't exist. Use `Try` to handle missing keys: -**Atomicity:** If the transform function raises, the store is unchanged. - -**Equivalent when the key already exists:** ```python +from doeff import Ok, Err + @do -def increment_counter_manual(): - current = yield Get("counter") +def safe_increment(): + result = yield Try(Get("counter")) + current = result.value if isinstance(result, Ok) else 0 + if isinstance(result, Err): + yield Put("counter", 0) new_value = current + 1 yield Put("counter", new_value) return new_value ``` -For missing keys, `Modify` and `Get` + `Put` are not equivalent because `Get` raises `KeyError` while `Modify` passes `None`. - ### State Composition Rules - `Put + Get` (immediate visibility): State changes are immediately visible within the same execution context - a `Put` followed by `Get` on the same key always returns the updated value. @@ -266,9 +277,9 @@ def counter_operations(): yield Put("count", 0) # Increment multiple times - yield Modify("count", lambda x: x + 1) - yield Modify("count", lambda x: x + 1) - yield Modify("count", lambda x: x + 1) + for _ in range(3): + val = yield Get("count") + yield Put("count", val + 1) # Read final value final = yield Get("count") @@ -283,8 +294,10 @@ def collect_items(): yield Put("items", []) # Add items - yield Modify("items", lambda xs: xs + [1]) - yield Modify("items", lambda xs: xs + [2, 3]) + items = yield Get("items") + yield Put("items", items + [1]) + items = yield Get("items") + yield Put("items", items + [2, 3]) # Read all items = yield Get("items") @@ -298,8 +311,8 @@ def state_machine(): yield Put("state", "idle") # Transition: idle -> processing - state = yield Get("state") - if state == "idle": + current_state = yield Get("state") + if current_state == "idle": yield Put("state", "processing") yield Tell("Started processing") @@ -320,7 +333,10 @@ Writer effects accumulate output (messages, events, structured entries) througho `Tell(message)` appends any Python object to the shared writer log: ```python -from doeff import do, Get, Tell, default_handlers, run +from doeff import do, run +from doeff_core_effects import Get, Tell +from doeff_core_effects.handlers import state, writer, writer_log +from doeff_core_effects.scheduler import scheduled @do def with_logging(): @@ -333,42 +349,51 @@ def with_logging(): yield Tell("Operation complete") return "done" +@do +def main_program(): + result = yield with_logging() + # result is "done" + # Access the writer log via writer_log() + log = yield writer_log() + return result, log + def main(): - result = run(with_logging(), default_handlers(), store={"count": 0}) - # All entries are in result.raw_store.get("__log__", []) + prog = main_program() + prog = writer(prog) + prog = state(initial={"count": 0})(prog) + result = run(scheduled(prog)) + print(result) main() ``` `Tell` is not string-only. It accepts and stores any Python object unchanged, including dictionaries, numbers, and custom objects. -### StructuredLog / slog - Structured Entries +### slog - Structured Entries -`StructuredLog(**entries)` logs a dictionary payload. -`slog(**entries)` is the lowercase alias. +`slog(msg, **kwargs)` logs a structured entry with a message and keyword arguments. +`WriterTellEffect(msg, **kwargs)` is the underlying effect class. ```python -def StructuredLog(**entries: object) -> Effect: - """Log a dictionary of key-value pairs.""" - -def slog(**entries: object) -> WriterTellEffect: - """Lowercase alias for StructuredLog.""" +def slog(msg, **kwargs) -> WriterTellEffect: + """Create a structured log entry with msg and keyword data.""" ``` ```python -from doeff import StructuredLog, do, slog +from doeff import do +from doeff_core_effects import slog @do def structured_logging(): - yield StructuredLog( + yield slog( + "User logged in", level="info", - message="User logged in", user_id=12345, ip="192.168.1.1", ) yield slog( + "High memory usage", level="warn", - message="High memory usage", memory_mb=512, threshold_mb=400, ) @@ -376,17 +401,18 @@ def structured_logging(): ### Listen - Capture Sub-Program Logs -`Listen(sub_program)` runs a sub-program and returns `ListenResult(value, log)`. +`Listen(sub_program)` runs a sub-program and returns `(value, collected)` -- a tuple of the sub-program's return value and the list of collected effects. Mechanism (SPEC-EFF-003): - Record the current log start index before running the sub-program. - Push an internal listen frame using that start index. - Execute the sub-program. -- Capture entries from that start index onward into the returned `ListenResult`. +- Capture entries from that start index onward into the returned collected list. - Keep those captured entries in the shared log store (they are not removed). ```python -from doeff import Listen, Tell, do +from doeff_core_effects import Listen, Tell +from doeff import do @do def inner_operation(): @@ -397,29 +423,20 @@ def inner_operation(): @do def outer_operation(): yield Tell("before inner") - listen_result = yield Listen(inner_operation()) + value, collected = yield Listen(inner_operation()) yield Tell("after inner") - return listen_result + return value, collected ``` -`ListenResult.log` is a `BoundedLog` (see `doeff.utils.BoundedLog`), not a plain `list`: - -```python -@dataclass -class ListenResult(Generic[T]): - value: T - log: BoundedLog -``` - -`BoundedLog` keeps bounded retention semantics: when capacity is exceeded, oldest entries are evicted. +The `collected` result is a plain list of captured effects. ### Listen Composition Rules -- `Listen + Tell`: entries told in the Listen scope appear in `ListenResult.log`. +- `Listen + Tell`: entries told in the Listen scope appear in `collected`. - `Listen + Local`: entries from inside `Local(...)` are captured normally by Listen. - `Listen + Try`: - - `Listen(Try(sub_program))` returns `ListenResult` whose `.value` is a `Result`; entries before the caught error remain in `.log`. - - `Try(Listen(sub_program))` returns `Err(...)` if `sub_program` fails before Listen completes; no `ListenResult` is produced on that path. + - `Listen(Try(sub_program))` returns `(result, collected)` where `result` is `Ok(value)` or `Err(error)`; entries before the caught error remain in `collected`. + - `Try(Listen(sub_program))` returns `Err(...)` if `sub_program` fails before Listen completes; no result tuple is produced on that path. - `Listen + Gather`: gathered programs share the same log store. - `Listen + Gather` in `SyncRuntime`: ordering is sequential in program order. - `Listen + Gather` in `AsyncRuntime`: ordering is non-deterministic and may interleave. @@ -436,7 +453,8 @@ def process_transaction(transaction_id): yield Put("balance", 1000) yield Tell(f"[AUDIT] Initial balance: 1000") - yield Modify("balance", lambda x: x - 100) + balance = yield Get("balance") + yield Put("balance", balance - 100) new_balance = yield Get("balance") yield Tell(f"[AUDIT] Debited 100, new balance: {new_balance}") @@ -482,7 +500,7 @@ def application_workflow(): # Process with retry logic for i in range(max_retries): attempt = yield Get("attempt") - yield Modify("attempt", lambda x: x + 1) + yield Put("attempt", attempt + 1) yield Tell(f"Attempt {attempt + 1}/{max_retries}") # Simulate work @@ -536,24 +554,25 @@ def isolated_operation(): yield Put("main_counter", 0) # Run sub-operation under Listen (captures logs, not state) - listen_result = yield Listen(isolated_sub_operation()) + value, collected = yield Listen(isolated_sub_operation()) # State changes inside Listen persist in the shared store main_count = yield Get("main_counter") yield Tell(f"Main counter after Listen: {main_count}") # 10 - return listen_result.value + return value @do def isolated_sub_operation(): # This mutates the same state store as the caller - yield Modify("main_counter", lambda x: x + 10) + val = yield Get("main_counter") + yield Put("main_counter", val + 10) yield Tell("Modified counter in sub-operation") return "sub-done" ``` `Listen` captures writer output (`Tell`) from the sub-program. It does not isolate state: -`Get`/`Put`/`Modify` effects inside `Listen(...)` remain visible after `Listen` completes. +`Get`/`Put` effects inside `Listen(...)` remain visible after `Listen` completes. ## Best Practices @@ -581,15 +600,16 @@ def get_database_config(): **DO:** - Use descriptive key names - Initialize state before reading -- Use `Modify` for atomic updates +- Use `Get` + `Put` for atomic updates ```python @do def safe_increment(): # Initialize if not exists - try: - count = yield Get("counter") - except KeyError: + result = yield Try(Get("counter")) + if isinstance(result, Ok): + count = result.value + else: yield Put("counter", 0) count = 0 @@ -605,19 +625,19 @@ def safe_increment(): **DO:** - Use consistent log formats - Log important state transitions -- Use `StructuredLog` for machine-readable logs +- Use `slog` for machine-readable logs ```python @do def well_logged_operation(): - yield StructuredLog( - event="operation_start", + yield slog( + "operation_start", timestamp=..., user_id=... ) # ... work ... - yield StructuredLog( - event="operation_complete", + yield slog( + "operation_complete", timestamp=..., duration_ms=... ) @@ -647,9 +667,8 @@ def well_logged_operation(): | `Local(env, prog)` | Scoped environment | Testing, overrides | | `Get(key)` | Read state | Counters, flags | | `Put(key, val)` | Write state | Initialize, update | -| `Modify(key, f)` | Transform state | Increment, append | | `Tell(msg)` | Append to log | Debugging, audit | -| `StructuredLog(**kw)` / `slog(**kw)` | Structured logging | Machine-readable logs | +| `slog(msg, **kw)` / `WriterTellEffect(msg, **kw)` | Structured logging | Machine-readable logs | | `Listen(prog)` | Capture sub-logs | Nested operations | ## Next Steps diff --git a/docs/04-async-effects.md b/docs/04-async-effects.md index 9d19f2cca..a7489c30b 100644 --- a/docs/04-async-effects.md +++ b/docs/04-async-effects.md @@ -4,7 +4,7 @@ This chapter covers async integration and scheduler primitives for cooperative c ## Table of Contents -- [Runner and Handler Presets](#runner-and-handler-presets) +- [Runner and Scheduled Programs](#runner-and-scheduled-programs) - [Await Effect](#await-effect) - [Scheduler Effect Catalog](#scheduler-effect-catalog) - [Waitables and Handles](#waitables-and-handles) @@ -16,32 +16,34 @@ This chapter covers async integration and scheduler primitives for cooperative c - [Common Mistakes](#common-mistakes) - [Quick Reference](#quick-reference) -## Runner and Handler Presets +## Runner and Scheduled Programs -Pair each runner with the matching handler preset. The preferred pairings are marked below. +There is a single runner: `run(doexpr)`. It takes one argument (a program expression) and returns the +raw result value directly. To enable scheduler effects (Spawn, Wait, Gather, Race, Cancel), wrap +your program with `scheduled()`. -The examples in this chapter use `handlers=` only to install the builtin sync/async runtime preset. For custom handler composition, call a Program -> Program handler installer directly, for example `handler(program)`. -| Runner | Preset | Status | Await Behavior | -| --- | --- | --- | --- | -| `run(...)` | `default_handlers()` | Valid (preferred) | Uses `sync_await_handler` | -| `await async_run(...)` | `default_async_handlers()` | Valid (preferred) | Uses `async_await_handler` | -| `await async_run(...)` | `default_handlers()` | Valid (non-preferred) | Works, but `Await` work runs sequentially/blocking | -| `run(...)` | `default_async_handlers()` | Invalid | Raises `TypeError` (`async_await_handler` requires async driver) | - ```python -from doeff import async_run, default_async_handlers, default_handlers, run +from doeff import run +from doeff_core_effects.scheduler import scheduled -sync_result = run(program(), handlers=default_handlers()) -async_result = await async_run(program(), handlers=default_async_handlers()) +result = run(scheduled(program())) +``` -# Non-preferred but valid: Await work is sequential/blocking. -slow_result = await async_run(program(), handlers=default_handlers()) +Handler composition example: -# Invalid pairing: raises TypeError. -bad_result = run(program(), handlers=default_async_handlers()) +```python +from doeff import run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled + +prog = my_program() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={"name": "doeff"})(prog) +result = run(scheduled(prog)) ``` ## Await Effect @@ -66,16 +68,15 @@ Both Await handlers bridge completion through `CreateExternalPromise` plus `Wait - submits the awaitable to a background asyncio loop thread - waits on `promise.future` via `Wait` - spawned Await work is sequential (no overlap) -- `async_await_handler` (`async_run` preset): +- `async_await_handler`: - schedules awaitable submission through the async runtime path - waits on `promise.future` via `Wait` - spawned Await work can overlap ### Await timing note -`Await` on sync `run(...)` is not concurrent by itself. If you spawn two Await tasks, each -`Await` resolves sequentially under `sync_await_handler`. Concurrency requires -`async_await_handler` with `async_run(...)`. +`Await` under `sync_await_handler` is not concurrent by itself. If you spawn two Await tasks, each +`Await` resolves sequentially. Concurrency requires the `async_await_handler`. ## Scheduler Effect Catalog @@ -87,7 +88,7 @@ The scheduler primitives are: | `Wait(task_or_future)` | `Task` or `Future` waitable handle | `T` | Suspend until one waitable resolves | | `Gather(*waitables)` | `Task`/`Future` waitable handles | `list[T]` | Suspend until all complete (input order) | | `Race(*waitables)` | `Task`/`Future` waitable handles | `RaceResult` | Suspend until first completion | -| `task.cancel()` | `Task[T]` | `None` | Request task cancellation | +| `Cancel(task)` | `Task[T]` | `None` | Request task cancellation (effect, not method) | | `SchedulerYield` | internal | internal | Cooperative preemption point inserted per yield | | `CreatePromise()` | none | `Promise[T]` | Allocate doeff-internal promise | | `CompletePromise(p, value)` | `Promise[T]`, `T` | `None` | Resolve promise successfully | @@ -206,10 +207,10 @@ def collect_all_even_on_errors(): - if that completion is cancellation, `Race` raises `TaskCancelledError` Race losers continue running by default. Cancel them explicitly if needed (for Task inputs): -`for t in result.rest: yield t.cancel()`. +`for t in result.rest: yield Cancel(t)`. ```python -from doeff import Race, Spawn, do +from doeff import Cancel, Race, Spawn, do @do def first_result(): @@ -221,7 +222,7 @@ def first_result(): # Race losers continue by default; cancel explicitly for teardown. for t in result.rest: - _ = yield t.cancel() + yield Cancel(t) return winner, value ``` @@ -229,20 +230,25 @@ def first_result(): Cancellation is explicit and cooperative: -- request cancellation via `yield task.cancel()` +- request cancellation via `yield Cancel(task)` - `Cancel` applies to `Pending`, `Running`, `Suspended`, and `Blocked` tasks - cancelling `Completed`/`Failed`/`Cancelled` tasks is a no-op - waiters (`Wait`, `Gather`, `Race`) observe cancelled tasks as `TaskCancelledError` - cancellation request returns immediately; running tasks cancel at next `SchedulerYield` ```python -from doeff import Try, Spawn, Wait, do +from doeff import Cancel, Ok, Err, Try, Spawn, Wait, do @do def cancel_child(): task = yield Spawn(work()) - _ = yield task.cancel() - return (yield Try(Wait(task))) # Err(TaskCancelledError) + yield Cancel(task) + result = yield Try(Wait(task)) + match result: + case Err(error=e): + return e # TaskCancelledError + case Ok(value=v): + return v ``` ## Promise vs ExternalPromise @@ -271,23 +277,21 @@ def complete_later(p): ## Common Mistakes -1. Passing `default_handlers()` to `async_run(...)` and expecting overlap. -This pairing is valid, but `Await` work runs sequentially under `sync_await_handler`. - -2. Passing `default_async_handlers()` to `run(...)`. -This pairing is invalid and raises `TypeError` because sync `run(...)` cannot handle async escape effects. - -3. Passing a raw coroutine to `Wait(...)`. +1. Passing a raw coroutine to `Wait(...)`. Use `Await(coroutine)` for Python async work; use `Wait` for scheduler waitables. -4. Passing raw programs directly to `Wait`, `Gather`, or `Race`. +2. Passing raw programs directly to `Wait`, `Gather`, or `Race`. Use `task = yield Spawn(program)` first, then pass the returned `Task` (or a `Future`) handle. -5. Expecting `Gather` to keep going after first failure. +3. Expecting `Gather` to keep going after first failure. Use `Try(...)` around child programs if you need partial success collection. -6. Assuming `Await` is concurrent under `run(...)`. -It is sequential under `sync_await_handler`; use `async_run(...)` for overlap. +4. Using `task.cancel()` instead of `yield Cancel(task)`. +`Cancel` is an effect that must be yielded; tasks do not have a `.cancel()` method. + +5. Forgetting to wrap the program with `scheduled()`. +Scheduler effects (`Spawn`, `Wait`, `Gather`, `Race`, `Cancel`) require +`run(scheduled(prog))`. ## Quick Reference @@ -298,7 +302,7 @@ It is sequential under `sync_await_handler`; use `async_run(...)` for overlap. | `Wait(waitable)` | waiting for one task/future handle | Accepts only `Task` or `Future` handles | | `Gather(*waitables)` | waiting for all spawned children | Pass `Task`/`Future` handles; fail-fast on first error/cancellation | | `Race(*waitables)` | waiting for first spawned child | Returns `RaceResult` (`first`, `value`, `rest`); losers continue unless cancelled | -| `task.cancel()` | requesting task cancellation | Applies to `Pending`/`Running`/`Suspended`/`Blocked`; terminal states are no-op | +| `Cancel(task)` | requesting task cancellation | Effect; yield it. Applies to `Pending`/`Running`/`Suspended`/`Blocked`; terminal states are no-op | | `SchedulerYield` | understanding scheduler fairness | Internal cooperative preemption point | | `CreatePromise()` | internal producer/consumer sync | Complete/fail via effects | | `CompletePromise(...)` | resolve internal promise | Wakes waiters | diff --git a/docs/05-error-handling.md b/docs/05-error-handling.md index 6883137fa..eb6b1d43f 100644 --- a/docs/05-error-handling.md +++ b/docs/05-error-handling.md @@ -1,28 +1,28 @@ # Error Handling -This chapter covers error handling in doeff using `RunResult`, the `Try` effect, and canonical `Ok`/`Err` result values. +This chapter covers error handling in doeff using the `Try` effect and canonical `Ok`/`Err` result values. ## Table of Contents -- [RunResult Overview](#runresult-overview) +- [Run Returns Raw Values](#run-returns-raw-values) - [Ok/Err Result Values](#okerr-result-values) -- [Result Methods](#result-methods) - [Pattern Matching](#pattern-matching) - [Try Effect](#try-effect) - [Try Composition Guarantees](#try-composition-guarantees) - [Captured Traceback on Err](#captured-traceback-on-err) - [Practical Patterns](#practical-patterns) -## RunResult Overview +## Run Returns Raw Values -`run()` and `async_run()` return a `RunResult[T]`. +`run(doexpr)` takes a single argument and returns the raw result value directly. There is no +`RunResult` wrapper. -Examples in this chapter pass `handlers=default_handlers()` only to install the builtin runtime -preset for Reader/State/Writer behavior. For custom handler composition, call a Program -> Program -handler installer directly, for example `handler(program)`. +For custom handler composition, call a Program -> Program handler installer directly, for example +`handler(program)`. ```python -from doeff import Ask, Tell, default_handlers, do, run +from doeff import Ask, Tell, do, run +from doeff_core_effects.handlers import reader, writer @do def program(): @@ -30,21 +30,15 @@ def program(): yield Tell(f"hello {name}") return name.upper() -result = run(program(), handlers=default_handlers(), env={"name": "doeff"}) - -if result.is_ok(): - print(result.value) # "DOEFF" - print(result.result) # Ok("DOEFF") -else: - print(result.error) +prog = program() +prog = writer(prog) +prog = reader(env={"name": "doeff"})(prog) +result = run(prog) +# result is "DOEFF" directly — no wrapper ``` -`RunResult` exposes: - -- `result`: canonical `Ok(value)` or `Err(error)` -- `value`: unwrap success value (raises on error) -- `error`: unwrap error (raises on success) -- `is_ok()` / `is_err()`: status checks +When an exception is raised inside the program, `run()` re-raises it directly. Use `Try` to +capture errors as `Ok`/`Err` values instead. ## Ok/Err Result Values @@ -54,8 +48,8 @@ Use the canonical import path: from doeff import Ok, Err ``` -`Ok` and `Err` values returned by doeff runtime surfaces (`RunResult.result`, `Try(...)`, task -completion payloads) are a unified Rust-backed Result implementation. +`Ok` and `Err` values are returned by `Try(...)` and task completion payloads. They are a Rust-backed +Result implementation. ### Fields @@ -68,20 +62,26 @@ completion payloads) are a unified Rust-backed Result implementation. - `bool(ok_value)` is `True` - `bool(err_value)` is `False` -## Result Methods +### Type checking + +Use `isinstance` to check the result type: -Use the Result API methods for transformations and fallbacks: +```python +from doeff import Ok, Err -- `value_or(default)`: get the success value or a fallback -- `map(f)`: transform only the `Ok` value -- `flat_map(f)`: chain functions that already return `Ok`/`Err` +if isinstance(result, Ok): + print(result.value) +elif isinstance(result, Err): + print(result.error) +``` ## Pattern Matching `Ok` and `Err` support native Python pattern matching. ```python -from doeff import Err, Ok, Try, default_handlers, do, run +from doeff import Err, Ok, Try, do, run +from doeff_core_effects.handlers import reader @do def might_fail(x: int): @@ -93,9 +93,11 @@ def might_fail(x: int): def workflow(x: int): return (yield Try(might_fail(x))) -result = run(workflow(-1), handlers=default_handlers()) +prog = workflow(-1) +prog = reader(env={})(prog) +result = run(prog) -match result.value: +match result: case Ok(value=v): print(f"success: {v}") case Err(error=e): @@ -107,7 +109,8 @@ match result.value: `Try(sub_program)` catches exceptions from `sub_program` and returns `Ok`/`Err` so you can keep control flow explicit. ```python -from doeff import Err, Ok, Try, Tell, default_handlers, do, run +from doeff import Err, Ok, Try, Tell, do, run +from doeff_core_effects.handlers import writer @do def parse_count(raw: str): @@ -125,8 +128,10 @@ def app(raw: str): yield Tell(f"parse failed: {exc}") return 0 -result = run(app("not-a-number"), handlers=default_handlers()) -print(result.value) # 0 +prog = app("not-a-number") +prog = writer(prog) +result = run(prog) +print(result) # 0 ``` ## Try Composition Guarantees @@ -137,7 +142,8 @@ print(result.value) # 0 State updates and `Tell(...)` log entries persist. ```python -from doeff import Get, Put, Try, default_handlers, do, run +from doeff import Err, Ok, Get, Put, Try, do, run +from doeff_core_effects.handlers import state @do def mutate_then_fail(): @@ -149,9 +155,11 @@ def app(): yield Put("counter", 0) result = yield Try(mutate_then_fail()) counter = yield Get("counter") - return (result.is_err(), counter) # (True, 1) + return (isinstance(result, Err), counter) # (True, 1) -print(run(app(), handlers=default_handlers()).value) +prog = app() +prog = state()(prog) +print(run(prog)) ``` ### 2. Env Restoration with `Local` @@ -161,7 +169,8 @@ both success and failure. This env restoration is independent from the no-rollba state/log. ```python -from doeff import Ask, Local, Try, default_handlers, do, run +from doeff import Ask, Local, Try, do, run +from doeff_core_effects.handlers import reader @do def failing_inner(): @@ -175,20 +184,23 @@ def app(): after = yield Ask("key") return (before, after) # ("outer", "outer") -print(run(app(), handlers=default_handlers(), env={"key": "outer"}).value) +prog = app() +prog = reader(env={"key": "outer"})(prog) +print(run(prog)) ``` `Local(Try(...))` is the inverse nesting order: `Try` catches inside the `Local` scope first, then `Local` restores the outer env when the scope exits. ```python -from doeff import Ask, Local, Try, default_handlers, do, run +from doeff import Ask, Local, Try, do, run +from doeff_core_effects.handlers import reader @do def local_safe_inner(): caught = yield Try(failing_inner()) still_inner = yield Ask("key") - return (caught.is_err(), still_inner) # (True, "inner") + return (isinstance(caught, Err), still_inner) # (True, "inner") @do def app_local_safe(): @@ -197,7 +209,9 @@ def app_local_safe(): after = yield Ask("key") return (before, inside, after) # ("outer", (True, "inner"), "outer") -print(run(app_local_safe(), handlers=default_handlers(), env={"key": "outer"}).value) +prog = app_local_safe() +prog = reader(env={"key": "outer"})(prog) +print(run(prog)) ``` Contrast: `Try(Local(...))` catches after `Local` unwinds; `Local(Try(...))` catches before unwind @@ -208,7 +222,7 @@ while still inside the local env. A nested Try does not collapse wrappers. `Try(Try(x))` returns nested results. ```python -from doeff import Try, default_handlers, do, run +from doeff import Ok, Err, Try, do, run @do def succeeds(): @@ -224,7 +238,7 @@ def app(): b = yield Try(Try(fails())) return (a, b) # (Ok(Ok(5)), Ok(Err(ValueError(...)))) -print(run(app(), handlers=default_handlers()).value) +print(run(app())) ``` ## Captured Traceback on Err @@ -232,7 +246,7 @@ print(run(app(), handlers=default_handlers()).value) `Err` includes `captured_traceback` for debugging context. ```python -from doeff import Try, default_handlers, do, run +from doeff import Ok, Err, Try, do, run @do def boom(): @@ -242,9 +256,9 @@ def boom(): def app(): return (yield Try(boom())) -safe_result = run(app(), handlers=default_handlers()).value +safe_result = run(app()) -if safe_result.is_err(): +if isinstance(safe_result, Err): print(type(safe_result.error).__name__) # ValueError print(safe_result.captured_traceback) # traceback object or None ``` @@ -254,7 +268,7 @@ if safe_result.is_err(): ### Explicit fallback ```python -from doeff import Try, default_handlers, do, run +from doeff import Ok, Err, Try, do, run @do def fetch_primary(): @@ -263,17 +277,19 @@ def fetch_primary(): @do def fetch_with_fallback(): first = yield Try(fetch_primary()) - if first.is_ok(): - return first.value - return "fallback" + match first: + case Ok(value=v): + return v + case _: + return "fallback" -print(run(fetch_with_fallback(), handlers=default_handlers()).value) +print(run(fetch_with_fallback())) ``` ### Keep failures visible ```python -from doeff import Try, default_handlers, do, run +from doeff import Ok, Err, Try, do, run @do def validate(v: int): @@ -286,6 +302,6 @@ def flow(v: int): checked = yield Try(validate(v)) return checked # caller receives Ok(...) or Err(...) -out = run(flow(0), handlers=default_handlers()).value -print(out.is_err()) # True +out = run(flow(0)) +print(isinstance(out, Err)) # True ``` diff --git a/docs/06-io-effects.md b/docs/06-io-effects.md index 0d3ad65a4..9f5d97735 100644 --- a/docs/06-io-effects.md +++ b/docs/06-io-effects.md @@ -11,8 +11,8 @@ here. `Spawn`, `Wait`, `Gather`, `Race`, `Try`, cache effects, graph effects, and semaphore effects. - For custom handler composition, call a Program -> Program handler installer directly, for example `handler(program)`. -- For builtin runtime behavior, install the sync or async preset with `default_handlers()` or - `default_async_handlers()`. +- For builtin runtime behavior, compose handlers individually — e.g. `writer(state()(prog))` — + and run with `run(doexpr)`. For async programs, use `run(scheduled(prog))`. - If you need to model a new side effect, define a domain-specific effect type and handle it with a dedicated handler installer instead of reaching for a generic `IO(...)` wrapper. diff --git a/docs/07-cache-system.md b/docs/07-cache-system.md index 3c8f9313d..a0e9cc81e 100644 --- a/docs/07-cache-system.md +++ b/docs/07-cache-system.md @@ -1,6 +1,6 @@ # Cache System -doeff provides a comprehensive caching system with policy-based cache management. +doeff provides a caching system with policy-based cache management via the `doeff_core_effects` package. > **Note**: For detailed cache system documentation including policy fields, lifecycle hints, storage options, and custom handlers, see **[cache.md](cache.md)**. @@ -11,28 +11,21 @@ doeff provides a comprehensive caching system with policy-based cache management - **`CacheGet(key)`** - Retrieve cached value (raises KeyError on miss) - **`CachePut(key, value, **policy)`** - Store value with policy hints -### Cache Decorator +### Import ```python -from doeff import cache, do - -@cache(ttl=60, lifecycle=CacheLifecycle.SESSION) -@do -def expensive_computation(x: int): - yield Tell("Computing (this should only happen once)...") - # Expensive work here - return x * 2 - -# First call computes -result1 = yield expensive_computation(5) # Computes - -# Second call uses cache -result2 = yield expensive_computation(5) # From cache +from doeff_core_effects.cache_effects import CacheGet, CachePut +from doeff_core_effects.cache_policy import CacheLifecycle, CacheStorage ``` ### Manual Cache Control ```python +from doeff import do +from doeff_core_effects import Tell +from doeff_core_effects.cache_effects import CacheGet, CachePut +from doeff_core_effects.cache_policy import CacheLifecycle, CacheStorage + @do def with_manual_cache(): try: @@ -59,9 +52,9 @@ def with_manual_cache(): | Field | Type | Purpose | |-------|------|---------| -| `ttl` | int | Time-to-live in seconds | -| `lifecycle` | CacheLifecycle | SESSION, PERSISTENT, TEMPORARY | -| `storage` | CacheStorage | MEMORY, DISK, DISTRIBUTED | +| `ttl` | float | Time-to-live in seconds | +| `lifecycle` | CacheLifecycle | TRANSIENT, SESSION, PERSISTENT | +| `storage` | CacheStorage | MEMORY, DISK | | `metadata` | dict | Custom metadata | See [cache.md](cache.md) for complete policy documentation. @@ -71,16 +64,26 @@ See [cache.md](cache.md) for complete policy documentation. ### API Response Caching ```python -@cache(ttl=60) +from doeff import do +from doeff_core_effects.cache_effects import CacheGet, CachePut + @do def fetch_user(user_id: int): - response = yield Await(httpx.get(f"/users/{user_id}")) - return response.json() + try: + return (yield CacheGet(f"user_{user_id}")) + except KeyError: + response = yield Await(httpx.get(f"/users/{user_id}")) + data = response.json() + yield CachePut(f"user_{user_id}", data, ttl=60) + return data ``` ### Conditional Caching ```python +from doeff import Try, do +from doeff_core_effects.cache_effects import CacheGet, CachePut + @do def smart_cache(key, fresh=False): if fresh: @@ -99,4 +102,4 @@ def smart_cache(key, fresh=False): - **[cache.md](cache.md)** - Complete cache system documentation - **[Patterns](12-patterns.md)** - Cache patterns and best practices -- **[Advanced Effects](09-advanced-effects.md)** - Gather and Atomic effects +- **[Advanced Effects](09-advanced-effects.md)** - Spawn, Gather, and concurrency effects diff --git a/docs/08-graph-tracking.md b/docs/08-graph-tracking.md deleted file mode 100644 index ca854f381..000000000 --- a/docs/08-graph-tracking.md +++ /dev/null @@ -1,204 +0,0 @@ -# Graph Tracking - -doeff can track program execution as a graph for visualization and debugging. - -## Graph Effects - -### Step - Add Named Step - -```python -@do -def with_steps(): - yield Step("initialize", "Setup phase") - yield Put("ready", True) - - yield Step("process", "Processing data") - result = yield process_data() - - yield Step("finalize", "Cleanup") - yield cleanup() - - return result -``` - -### Annotate - Add Metadata - -```python -@do -def with_annotations(): - yield Annotate({"user_id": 123, "operation": "fetch"}) - data = yield fetch_data() - yield Annotate({"records": len(data)}) - return data -``` - -### Snapshot - Capture Current Graph - -```python -@do -def with_snapshot(): - yield Step("start") - - # Capture graph at this point - graph = yield Snapshot() - yield Tell(f"Graph has {len(graph.steps)} steps") - - yield Step("continue") - return "done" -``` - -### CaptureGraph - Get Final Graph - -```python -from doeff import run, default_handlers - -@do -def traced_program(): - yield Step("step1") - yield Step("step2") - yield Step("step3") - return "result" - -result = run(traced_program(), default_handlers()) -# Graph is tracked during execution in result.raw_store.get("__graph__") -``` - -## Visualization - -### Export to HTML - -```python -from doeff import graph_to_html, run, default_handlers - -# Run program with graph tracking -result = run(my_program(), default_handlers()) - -# Get graph from raw_store -graph = result.raw_store.get("__graph__") -if graph: - # graph_to_html is a @do function, run it to get HTML - html_result = run(graph_to_html(graph), default_handlers()) - html = html_result.value -``` - -### Getting the Graph - -```python -from doeff import run, default_handlers - -result = run(my_program(), default_handlers()) - -# Access graph from raw_store -graph = result.raw_store.get("__graph__") -``` - -## Use Cases - -### Debugging Complex Workflows - -```python -@do -def complex_workflow(): - yield Step("load_config") - config = yield load_config() - yield Annotate({"config_keys": list(config.keys())}) - - yield Step("validate_inputs") - valid = yield validate(config) - yield Annotate({"validation_passed": valid}) - - if valid: - yield Step("process_data") - result = yield process(config) - yield Annotate({"result_size": len(result)}) - else: - yield Step("error_handling") - result = [] - - yield Step("complete") - return result -``` - -### Performance Analysis - -```python -from doeff_time import GetTime - -@do -def timed_operations(): - yield Step("operation_a") - start = yield GetTime() - yield expensive_a() - end = yield GetTime() - duration_a = (end - start).total_seconds() - yield Annotate({"duration_a": duration_a}) - - yield Step("operation_b") - start = yield GetTime() - yield expensive_b() - end = yield GetTime() - duration_b = (end - start).total_seconds() - yield Annotate({"duration_b": duration_b}) -``` - -## Graph Structure - -The execution graph is a `WGraph` with: - -```python -@dataclass -class WGraph: - last: WStep # Most recent step - steps: frozenset[WStep] # All steps - -@dataclass -class WStep: - inputs: tuple[WNode, ...] # Input nodes - output: WNode # Output node - meta: dict # Metadata -``` - -## Best Practices - -### Strategic Step Placement - -```python -# Good: steps at logical boundaries -@do -def well_traced(): - yield Step("phase1") - yield do_phase1() - - yield Step("phase2") - yield do_phase2() - -# Less useful: too granular -@do -def over_traced(): - yield Step("increment") - x = yield Get("x") - yield Step("add_one") - y = x + 1 - yield Step("store") - yield Put("x", y) -``` - -### Meaningful Annotations - -```python -# Good: useful context -yield Annotate({ - "user_id": user_id, - "records_processed": count, - "cache_hit_rate": hits/total -}) - -# Less useful: redundant info -yield Annotate({"step": "processing"}) # Use Step instead -``` - -## Next Steps - -- **[Patterns](12-patterns.md)** - Graph tracking patterns -- **[Advanced Effects](09-advanced-effects.md)** - Gather for parallel execution -- **[MARKERS.md](MARKERS.md)** - Marker system for Program manipulation diff --git a/docs/09-advanced-effects.md b/docs/09-advanced-effects.md index efc72b189..2f37a18aa 100644 --- a/docs/09-advanced-effects.md +++ b/docs/09-advanced-effects.md @@ -4,21 +4,22 @@ Advanced effects for parallel execution, shared-state coordination, and runtime- ## Gather Effects -Execute multiple Programs in parallel and collect results. +Execute multiple spawned tasks in parallel and collect results. -### Gather - Parallel Programs +### Gather - Parallel Tasks ```python -from doeff import Gather, do +from doeff import do +from doeff_core_effects.scheduler import Spawn, Gather, Wait, scheduled @do def parallel_programs(): - prog1 = fetch_user(1) - prog2 = fetch_user(2) - prog3 = fetch_user(3) + t1 = yield Spawn(fetch_user(1)) + t2 = yield Spawn(fetch_user(2)) + t3 = yield Spawn(fetch_user(3)) - # Run all Programs in parallel - users = yield Gather(prog1, prog2, prog3) + # Wait for all tasks in parallel + users = yield Gather(t1, t2, t3) # users = [user1, user2, user3] return users @@ -26,7 +27,7 @@ def parallel_programs(): ### User-Side Dict Pattern -If you need to run a dict of Programs in parallel, use `Gather` with dict reconstruction: +If you need to run a dict of programs in parallel, use `Spawn` + `Gather` with dict reconstruction: ```python @do @@ -37,7 +38,10 @@ def parallel_dict(): "comments": fetch_comments(123), } keys = list(programs.keys()) - values = yield Gather(*programs.values()) + tasks = [] + for prog in programs.values(): + tasks.append((yield Spawn(prog))) + values = yield Gather(*tasks) results = dict(zip(keys, values)) # results = {"user": ..., "posts": [...], "comments": [...]} @@ -46,10 +50,11 @@ def parallel_dict(): ### Using Gather with Async Operations -To run async operations in parallel, wrap them with `Await`: +To run async operations in parallel, wrap them with `Await` and `Spawn`: ```python -from doeff import Await, Gather, do +from doeff import Await, do +from doeff_core_effects.scheduler import Spawn, Gather @do def parallel_async(): @@ -57,10 +62,10 @@ def parallel_async(): def fetch_data(url): return (yield Await(http_get(url))) - results = yield Gather( - fetch_data("https://api1.example.com"), - fetch_data("https://api2.example.com"), - ) + t1 = yield Spawn(fetch_data("https://api1.example.com")) + t2 = yield Spawn(fetch_data("https://api2.example.com")) + + results = yield Gather(t1, t2) return results ``` @@ -77,51 +82,40 @@ Use `Try(...)` around child programs when you need partial results instead of fa ## State Effects in Concurrent Programs -State semantics use `Get`, `Put`, and `Modify`. +State semantics use `Get` and `Put`. - `Get(key)`: read a required value (raises `KeyError` if missing) - `Put(key, value)`: write a value -- `Modify(key, fn)`: atomic read-modify-write update -### Atomic Updates with Modify +### Read-Modify-Write with Get + Put -Use `Modify` when multiple tasks can update the same key. +Use `Get` followed by `Put` when you need to update state: ```python -from doeff import Modify, Tell, do +from doeff import do +from doeff_core_effects import Get, Put, Tell @do def increment_counter(): - new_value = yield Modify("counter", lambda current: (current or 0) + 1) + current = yield Get("counter") + new_value = (current or 0) + 1 + yield Put("counter", new_value) yield Tell(f"Counter: {new_value}") return new_value ``` -### Get/Put vs Modify - -```python -from doeff import Get, Modify, Put, do - -@do -def unsafe_increment(): - # Not atomic across tasks - count = yield Get("counter") - yield Put("counter", count + 1) - -@do -def safe_increment(): - # Atomic read-modify-write - return (yield Modify("counter", lambda current: (current or 0) + 1)) -``` - -### Common Modify Patterns +### Common Update Patterns ```python -from doeff import Modify, do +from doeff import do +from doeff_core_effects import Get, Put @do def accumulate_result(value): - return (yield Modify("total", lambda current: (current or 0) + value)) + current = yield Get("total") + new_total = (current or 0) + value + yield Put("total", new_total) + return new_total ``` ## Semaphore Effects @@ -164,7 +158,7 @@ def run_workers(): tasks = [] for i in range(10): tasks.append((yield Spawn(worker(sem, i)))) - return (yield Gather(*[Wait(task) for task in tasks])) + return (yield Gather(*tasks)) ``` ### Semaphore Use Cases @@ -183,7 +177,7 @@ def run_workers(): ## Spawn Effect -Execute Programs in the background and retrieve results later. +Execute programs in the background and retrieve results later. ### Basic Spawn @@ -222,7 +216,7 @@ def parallel_background_work(): | Effect | Execution | Use Case | |--------|-----------|----------| -| `Gather(*progs)` | Parallel, blocking | Wait for all immediately | +| `Gather(*tasks)` | Parallel, blocking | Wait for all spawned tasks immediately | | `Spawn(prog)` | Background, non-blocking | Do other work while waiting | ```python @@ -230,7 +224,11 @@ from doeff import Gather, Spawn, Wait, do @do def comparison(): - results = yield Gather(prog1(), prog2(), prog3()) + # Spawn all, then gather + t1 = yield Spawn(prog1()) + t2 = yield Spawn(prog2()) + t3 = yield Spawn(prog3()) + results = yield Gather(t1, t2, t3) task = yield Spawn(slow_prog()) yield do_other_work() @@ -240,45 +238,47 @@ def comparison(): ## Race Effect -`Race(*waitables)` resumes when the first waitable completes. +`Race(*tasks)` resumes when the first task completes. - First completion wins. - If the winner fails, `Race` raises that error. - If the winner is cancelled, `Race` raises `TaskCancelledError`. -- The returned `RaceResult` exposes `first`, `value`, and `rest`. +- `Race` returns the winning value directly. By default, non-winning siblings keep running. Cancel them explicitly when you want winner-takes-all behavior: ```python -from doeff import Race, Spawn, do +from doeff import Cancel, Race, Spawn, do @do def first_wins(): fast = yield Spawn(fetch_fast()) slow = yield Spawn(fetch_slow()) - result = yield Race(fast, slow) - for loser in result.rest: - _ = yield loser.cancel() + value = yield Race(fast, slow) + + # Cancel remaining tasks explicitly + yield Cancel(fast) + yield Cancel(slow) - return result.value + return value ``` ## Cancel / TaskCancelledError Cancellation is cooperative and non-blocking: -- `yield task.cancel()` requests cancellation and returns immediately. +- `yield Cancel(task)` requests cancellation and returns immediately. - `Wait`, `Gather`, and `Race` raise `TaskCancelledError` when waiting on a cancelled task. ```python -from doeff import Try, Spawn, TaskCancelledError, Wait, do +from doeff import Cancel, Try, Spawn, TaskCancelledError, Wait, do @do def cancel_and_join(): task = yield Spawn(long_running_job()) - _ = yield task.cancel() + yield Cancel(task) joined = yield Try(Wait(task)) if joined.is_err() and isinstance(joined.error, TaskCancelledError): @@ -286,64 +286,52 @@ def cancel_and_join(): return joined.value ``` -## WithIntercept +## WithObserve -`WithIntercept` is the canonical interception primitive. It observes or rewrites yielded effects +`WithObserve` is the canonical observation primitive. It observes yielded effects within a scoped subtree (including child dispatch in that scope). -### Interceptor Contract +### Observer Contract ```python -from doeff import Effect, Program, do +from doeff import do -@do -def interceptor(effect: Effect) -> Effect | Program: - ... +def observer(effect): + # Observer is a plain callable that receives each effect. + # It is called for observation only — return value is ignored. + pass ``` -- Return the original effect to pass through unchanged. -- Return a replacement `Effect` or `Program` to rewrite that dispatch step. -- If the interceptor raises, that error propagates. -- Program exceptions inside the intercepted subtree still propagate unless wrapped (for example by `Try`). - ### Child Propagation -`WithIntercept` applies to scoped child execution contexts: +`WithObserve` applies to scoped child execution contexts: -- `Gather` branches run under the intercept scope. +- `Gather` branches run under the observe scope. - `Spawn` tasks created inside the scope inherit the scope. In parallel contexts, observation order is not guaranteed. ```python -from doeff import Ask, AskEffect, Gather, Spawn, WithIntercept, do - -@do -def fallback_user(): - value = yield Ask("fallback_user") - return f"user:{value}" - -@do -def interceptor(effect): - if isinstance(effect, AskEffect) and effect.key == "user": - return fallback_user() - return effect +from doeff import Ask, Spawn, WithObserve, Gather, Wait, do @do def child(): return (yield Ask("user")) @do -def run_with_intercept(): - t1 = yield Spawn(child()) - t2 = yield Spawn(child()) - return (yield WithIntercept(interceptor, Gather(t1, t2), types=(AskEffect,), mode="include")) -``` +def run_with_observe(): + seen = [] + def my_observer(effect): + seen.append(effect) -### Nested Interceptors + @do + def body(): + t1 = yield Spawn(child()) + t2 = yield Spawn(child()) + return (yield Gather(t1, t2)) -```python -WithIntercept(outer, WithIntercept(inner, program(), types=(), mode="exclude")) + result = yield WithObserve(my_observer, body()) + return result ``` ## Custom Control Effects with Frames @@ -364,7 +352,11 @@ retries without modifying the runtime internals. ## Time Effects Runtime Matrix -`Delay`, `GetTime`, and `WaitUntil` are runtime-dependent. Use this behavior matrix: +`Delay`, `GetTime`, and `WaitUntil` are runtime-dependent. They live in the `doeff_time` package: + +```python +from doeff_time import Delay, GetTime, WaitUntil +``` | Effect | Sync Runtime | Simulation Runtime | Async Runtime | |--------|--------------|--------------------|---------------| @@ -376,10 +368,11 @@ If `WaitUntil` receives a time in the past, it returns immediately. ## Combining Advanced Effects -### Gather + Modify +### Gather + State Updates ```python -from doeff import Gather, Get, Modify, Put, do +from doeff import Gather, Spawn, do +from doeff_core_effects import Get, Put @do def parallel_counter(): @@ -388,17 +381,17 @@ def parallel_counter(): @do def increment_task(n): for _ in range(n): - yield Modify("count", lambda current: (current or 0) + 1) + current = yield Get("count") + yield Put("count", (current or 0) + 1) return "done" - yield Gather( - increment_task(100), - increment_task(100), - increment_task(100), - ) + t1 = yield Spawn(increment_task(100)) + t2 = yield Spawn(increment_task(100)) + t3 = yield Spawn(increment_task(100)) + yield Gather(t1, t2, t3) final = yield Get("count") - return final # 300 + return final ``` ## Best Practices @@ -422,20 +415,20 @@ This does not require adding `__interpreter__` to your environment. ### When to Use Gather **DO:** -- Multiple independent Programs +- Multiple independent spawned tasks - Fan-out computation patterns - Parallel data fetching **DON'T:** - Dependent computations (use sequential yields) -- Single Program (just yield it directly) +- Single program (just yield it directly) -### When to Use Modify +### When to Use Get + Put **DO:** -- Concurrent state updates +- State reads and updates - Counters and accumulators -- Atomic read-modify-write flows +- Sequential read-modify-write flows **DON'T:** - Isolated single-step reads (`Get` is enough) @@ -452,32 +445,30 @@ This does not require adding `__interpreter__` to your environment. - Skip `ReleaseSemaphore` on error paths - Use huge permit counts as a substitute for backpressure design -### When to Use WithIntercept +### When to Use WithObserve **DO:** -- Rewrite or short-circuit selected effects in a scoped subtree -- Inject fallback programs for missing data or policy checks -- Use `types`/`mode` filters to scope observation cost +- Observe effects in a scoped subtree for logging or tracing +- Monitor effects across parallel child branches **DON'T:** - Assume deterministic observation order across parallel child effects -- Return replacement programs that recursively trigger the same interceptor without a stop condition ## Summary | Effect | Purpose | Use Case | |--------|---------|----------| -| `Gather(*progs)` | Parallel Programs | Fan-out computation | +| `Spawn(prog)` + `Gather(*tasks)` | Parallel tasks | Fan-out computation | | `Spawn(prog)` | Background execution | Non-blocking tasks | -| `Race(*waitables)` | First-completion wait | Fastest-response selection | -| `task.cancel()` | Cooperative task cancellation | Stop non-winning/background work | -| `WithIntercept(f, expr, types, mode)` | Scoped effect transformation | Policy injection, tracing, effect rewriting | -| `Modify(key, fn)` | Atomic read-modify-write | Shared-state updates | +| `Race(*tasks)` | First-completion wait | Fastest-response selection | +| `Cancel(task)` | Cooperative task cancellation | Stop non-winning/background work | +| `WithObserve(observer, body)` | Scoped effect observation | Tracing, logging | +| `Get(key)` + `Put(key, val)` | State read and write | Shared-state updates | | `CreateSemaphore / AcquireSemaphore / ReleaseSemaphore` | Cooperative concurrency limit | Rate limiting, mutex, pools | -| `Delay / GetTime / WaitUntil` | Runtime-aware time control | Time-based workflows | +| `Delay / GetTime / WaitUntil` | Runtime-aware time control (doeff_time) | Time-based workflows | ## Next Steps -- **[Async Effects](04-async-effects.md)** - Async operations with Await and Gather +- **[Async Effects](04-async-effects.md)** - Async operations with Await - **[Cache System](07-cache-system.md)** - Persistent caching - **[Patterns](12-patterns.md)** - Advanced patterns and best practices diff --git a/docs/11-kleisli-arrows.md b/docs/11-kleisli-arrows.md index 4f391209b..cc4d36de6 100644 --- a/docs/11-kleisli-arrows.md +++ b/docs/11-kleisli-arrows.md @@ -1,82 +1,59 @@ # Kleisli Arrows -`@do` produces a `KleisliProgram[P, T]`. Calling that `KleisliProgram` performs -pure call-time macro expansion and emits a `Call` DoCtrl directly. +`@do` produces a normal callable. Calling that callable performs +pure call-time macro expansion and emits an `Expand` DoExpr directly. ## Table of Contents - [What `@do` Returns](#what-do-returns) - [Call-Time Macro Expansion](#call-time-macro-expansion) -- [Why Call Is a Macro](#why-call-is-a-macro) +- [Why Expand Is a Macro](#why-expand-is-a-macro) - [KPC Non-Effect Invariant](#kpc-non-effect-invariant) - [Annotation-Aware Argument Classification](#annotation-aware-argument-classification) - [`@do` Handler Authoring Contract](#do-handler-authoring-contract) -- [Call Metadata](#call-metadata) -- [Composability of `Call` DoCtrl](#composability-of-call-doctrl) -- [Kleisli-Level Composition and Partial Application](#kleisli-level-composition-and-partial-application) +- [Composition via Nested `@do`](#composition-via-nested-do) - [Migration Note](#migration-note) - [Best Practices](#best-practices) ## What `@do` Returns ```python -from doeff import Program, do +from doeff import do @do def add_one(x: int): return x + 1 call_expr = add_one(41) -# call_expr is a Call DoCtrl (Program-shaped DoExpr). +# call_expr is an Expand DoExpr (program-shaped DoExpr). ``` Conceptually: -- `@do` transforms a function into `KleisliProgram[P, T]` -- `KleisliProgram.__call__` returns `Call[...]` -- the VM evaluates that `Call` directly +- `@do` transforms a function into a normal callable +- Calling it returns an `Expand` object +- The VM evaluates that `Expand` directly ## Call-Time Macro Expansion -When you call a `KleisliProgram`, expansion happens synchronously at Python call +When you call a `@do`-decorated function, expansion happens synchronously at Python call time: 1. Load cached auto-unwrap strategy (computed once at decoration time). 2. Classify each argument using annotation-aware `should_unwrap`. 3. Convert each argument to a DoExpr node. -4. Populate call metadata. -5. Return `Call(Pure(kernel), args, kwargs, metadata)`. +4. Return an `Expand(...)` DoExpr. No handlers are consulted during this expansion. The transformation is pure. -### Expansion Pseudocode +## Why Expand Is a Macro -```python -def __call__(self, *args, **kwargs): - strategy = self._auto_unwrap_strategy - do_expr_args = [classify(arg, strategy.should_unwrap_positional(i)) for i, arg in enumerate(args)] - do_expr_kwargs = { - name: classify(value, strategy.should_unwrap_keyword(name)) - for name, value in kwargs.items() - } - - metadata = { - "function_name": self.__name__, - "source_file": self.original_func.__code__.co_filename, - "source_line": self.original_func.__code__.co_firstlineno, - "program_call": None, - } - return Call(Pure(self.execution_kernel), do_expr_args, do_expr_kwargs, metadata) -``` - -## Why Call Is a Macro - -`KleisliProgram.__call__` is intentionally a macro step to preserve phase separation: +Calling a `@do`-decorated function is intentionally a macro step to preserve phase separation: Python call-time expansion builds `DoExpr`, and VM runtime evaluation executes `DoExpr`. Treating call-construction as an effect introduces a recursion flaw where call dispatch must invoke effect handling before the call tree is fully constructed. -The macro model avoids that cycle by constructing `Call(...)` immediately, then letting +The macro model avoids that cycle by constructing `Expand(...)` immediately, then letting standard DoExpr evaluation proceed. Historical removed component matrix details are kept in `docs/revision-log.md` so this chapter stays focused on the current model. @@ -86,9 +63,9 @@ in `docs/revision-log.md` so this chapter stays focused on the current model. > > - It does not extend `PyEffectBase`. > - It is never dispatched through `Perform(KPC(...))`. -> - There is no KPC handler in `default_handlers()` or runtime handler stacks. +> - There is no KPC handler in the runtime handler stack. -Calling a `KleisliProgram` returns a `Call` DoCtrl directly, and the VM evaluates that +Calling a `@do`-decorated function returns an `Expand` DoExpr directly, and the VM evaluates that DoExpr normally. ## Annotation-Aware Argument Classification @@ -123,16 +100,17 @@ DoExpr normally. ### Example ```python -from doeff import Ask, Program, do +from doeff import Ask, Pure, do +from doeff import Program @do -def use_values(x: int, raw_program: Program[int]): +def use_values(x: int, raw_program: Program): y = yield raw_program return f"{x}:{y}" -call_expr = use_values(Ask("x"), Program.pure(10)) +call_expr = use_values(Ask("x"), Pure(10)) # x: int -> should_unwrap=True -> Ask("x") becomes Perform(Ask("x")) -# raw_program: Program[int] -> should_unwrap=False -> Program.pure(10) becomes Pure(...) +# raw_program: Program -> should_unwrap=False -> Pure(10) becomes Pure(...) ``` ## `@do` Handler Authoring Contract @@ -148,56 +126,15 @@ signature contract: If these rules are violated, handler installation fails with `TypeError` at the Python API boundary. -## Call Metadata - -`KleisliProgram.__call__` attaches metadata at call time. The fields are used for -tracing and stack introspection: - -- `function_name` -- `source_file` -- `source_line` -- `program_call` (optional call context object) - -Example access: - -```python -@do -def compute(x: int): - return x * 2 - -call_expr = compute(21) -meta = call_expr.meta -print(meta["function_name"]) # compute -``` - -## Composability of `Call` DoCtrl +## Composition via Nested `@do` -A `Call` is a DoExpr node, so it composes like any other DoExpr. +Since `@do` functions return `Expand` objects with no `.map()`, `.flat_map()`, `.fmap()`, +`.partial()`, or `>>` operators, composition is done via nested `@do` functions and `yield`: -Semantic shape (SPEC-KPC-001): +### Sequential Composition ```python -result = fetch_user(42) # Call DoCtrl -mapped = fetch_user(42).map(lambda u: u.name) # Map(Call(...), f) -chained = fetch_user(42).flat_map(enrich) # FlatMap(Call(...), f) -value = yield fetch_user(42) # yield sends Call to VM -``` - -The important invariant is the expression shape: - -- `fetch_user(42)` returns `Call(...)` -- mapping/chaining over it yields `Map(Call(...), ...)` or - `FlatMap(Call(...), ...)` -- `run()` evaluates the resulting DoExpr tree directly - -## Kleisli-Level Composition and Partial Application - -Kleisli-level combinators still work and remain useful: - -### `and_then_k` / `>>` - -```python -from doeff import default_handlers, do, run +from doeff import do, run @do def fetch_user(user_id: int): @@ -207,28 +144,38 @@ def fetch_user(user_id: int): def fetch_posts(user: dict): return [f"post-for-{user['name']}"] -fetch_user_posts = fetch_user >> fetch_posts -result = run(fetch_user_posts(7), default_handlers()) +@do +def fetch_user_posts(user_id: int): + user = yield fetch_user(user_id) + posts = yield fetch_posts(user) + return posts + +result = run(fetch_user_posts(7)) ``` -### `fmap` +### Mapping Over Results ```python @do def get_user(): return {"id": 1, "name": "Alice"} -get_name = get_user.fmap(lambda user: user["name"]) +@do +def get_name(): + user = yield get_user() + return user["name"] ``` -### `partial` +### Partial Application via Closure ```python @do def greet(prefix: str, name: str): return f"{prefix}, {name}" -say_hello = greet.partial("Hello") +@do +def say_hello(name: str): + return (yield greet("Hello", name)) ``` ### Varargs Auto-Unwrap at Composition Boundaries @@ -245,11 +192,6 @@ Boundary rules to keep in mind: varargs are resolved at that call boundary. - Annotating varargs as `Program[...]` or `Effect[...]` sets `should_unwrap=False`, so those values are passed through as data. -- `partial(...)` does not bypass classification. Pre-bound arguments and call-time - arguments are merged, then classified by the base Kleisli signature. - -This is why varargs-heavy composition can feel surprising: every Kleisli call -boundary applies its own annotation-driven unwrap policy. ## Migration Note @@ -261,20 +203,21 @@ This chapter documents only the current call-time macro architecture. - Prefer explicit annotations for parameters that should not auto-unwrap. - Use `Program[...]`/`Effect[...]` annotations when you need raw objects in the function body. -- Treat `KleisliProgram.__call__` as macro expansion that constructs a DoExpr +- Treat calling a `@do`-decorated function as macro expansion that constructs a DoExpr tree for VM evaluation. -- Keep reasoning at the DoExpr level: each call produces a `Call` node. +- Keep reasoning at the DoExpr level: each call produces an `Expand` node. +- Use nested `@do` functions and `yield` for composition instead of method chaining. ## Summary | Topic | Macro Model | | --- | --- | | KPC identity | Call-time macro | -| `__call__` result | `Call` DoCtrl | +| `__call__` result | `Expand` DoExpr | | Resolution path | Pure expansion + VM eval | -| Handler dependency | `Call` executes as regular DoExpr | +| Handler dependency | `Expand` executes as regular DoExpr | | Argument policy | Annotation-aware `should_unwrap` | -| Metadata | Populated at call time | +| Composition | Nested `@do` + `yield` | ## Next Steps diff --git a/docs/12-mcp-tools.md b/docs/12-mcp-tools.md index 5a708a5d9..a54b89f10 100644 --- a/docs/12-mcp-tools.md +++ b/docs/12-mcp-tools.md @@ -197,7 +197,8 @@ from pathlib import Path from doeff import do, run, Perform from doeff.mcp import McpToolDef, McpParamSchema from doeff_agents.adapters.base import AgentType -from doeff_agents.effects import LaunchEffect, Monitor, Sleep, Capture, Stop +from doeff_agents.effects import LaunchEffect, Monitor, Capture, Stop +from doeff_agents.sessionhost.effects import ClockSleep from doeff_agents.handlers import claude_agent_handler # 1. Define a tool @@ -225,7 +226,7 @@ def main(): # Monitor until done for _ in range(60): - yield Sleep(1.0) + yield ClockSleep(seconds=1.0) obs = yield Monitor(handle) if obs.is_terminal: break @@ -244,7 +245,8 @@ print(result) ```hy (require doeff-hy.macros [defmcp-tool defp <- defhandler]) (import doeff [run]) -(import doeff_agents.effects [Launch Monitor Sleep Capture Stop]) +(import doeff_agents.effects [Launch Monitor Capture Stop]) +(import doeff_agents.sessionhost.effects [ClockSleep]) (import doeff_agents.handlers [claude-agent-handler]) (import doeff_agents.adapters.base [AgentType]) (import pathlib [Path]) diff --git a/docs/12-patterns.md b/docs/12-patterns.md index 5a48478a3..e680dda9b 100644 --- a/docs/12-patterns.md +++ b/docs/12-patterns.md @@ -9,6 +9,9 @@ Common patterns, best practices, and anti-patterns for building robust applicati Separate concerns into distinct layers: ```python +from doeff import do, Try +from doeff_core_effects import Ask, Tell + # Domain layer - pure business logic @do def calculate_order_total(items): @@ -52,6 +55,9 @@ def get_order_handler(order_id): Centralize data access: ```python +from doeff import do +from doeff_core_effects import Ask + @do def create_repository(table_name): db = yield Ask("database") @@ -100,6 +106,9 @@ def app(): Centralized dependency access: ```python +from doeff import do +from doeff_core_effects import Ask + @do def create_services(): return { @@ -123,6 +132,9 @@ def business_logic(): Transactional boundary: ```python +from doeff import do +from doeff_core_effects import Ask, Tell + @do def unit_of_work(operations): db = yield Ask("database") @@ -163,12 +175,12 @@ This section summarizes the composition guarantees from ### Effect Combination Matrix -| Outer / Inner | Ask | Get | Put | Tell | Try | Local | Listen | WithIntercept | Gather | +| Outer / Inner | Ask | Get | Put | Tell | Try | Local | Listen | WithObserve | Gather | |---------------|-----|-----|-----|------|------|-------|--------|-----------|--------| | **Local** | Scoped | Propagates | Propagates | Propagates | Propagates | Scoped | Propagates | Propagates | Propagates | | **Listen** | - | - | - | Captured* | - | - | Nested | - | All captured* | | **Try** | - | - | Persists | Persists | Wrapped | Restores | - | - | First error | -| **WithIntercept** | Transform | Transform | Transform | Transform | Transform | Transform | Transform | Transform | Transform | +| **WithObserve** | Observe | Observe | Observe | Observe | Observe | Observe | Observe | Observe | Observe | | **Gather** | Inherit | Shared** | Shared** | Merged | Isolated | Inherit | Shared | Propagates | Nested | Matrix key: @@ -178,7 +190,7 @@ Matrix key: - `Persists`: Side effects persist even when errors occur. - `Wrapped`: Result is wrapped in the outer effect's result type. - `Restores`: Environment context is restored after inner completion. -- `Transform`: Effect may be transformed by the outer effect. +- `Observe`: Effect is observed by the observer function. - `Isolated`: Branch-local error handling boundaries apply. - `Inherit`: Child inherits parent context snapshot at invocation time. - `Shared`: Branches share the same underlying store/runtime resource. @@ -197,7 +209,7 @@ The following laws from SPEC-EFF-100 apply to all compositions: 3. **Listen Capture Law**: `Listen` captures `Tell` output ONLY on success; on error it propagates the error. 4. **Try Non-Rollback Law**: `Try` does NOT roll back state on error. 5. **Try Environment Restoration Law**: `Try` restores environment context, but not state. -6. **WithIntercept Transformation Law**: `WithIntercept` transforms effects in nested programs, including `Gather` children. +6. **WithObserve Observation Law**: `WithObserve` observes effects in nested programs, including `Gather` children. 7. **Gather Environment Inheritance Law**: `Gather` children inherit parent environment snapshot at invocation. 8. **Gather Store Sharing Law**: Store sharing under `Gather` is runtime-dependent. 9. **Gather Error Propagation Law**: First child error propagates; sibling behavior is runtime-dependent. @@ -217,17 +229,18 @@ The following laws from SPEC-EFF-100 apply to all compositions: Wrap operations with consistent error handling: ```python +from doeff import Try, do +from doeff_core_effects import Tell + @do def with_error_handling(operation, operation_name): yield Tell(f"Starting: {operation_name}") - yield Step(f"start_{operation_name}") safe_result = yield Try(operation()) if safe_result.is_err(): yield handle_error(operation_name, safe_result.error) result = safe_result.value if safe_result.is_ok() else None - yield Step(f"end_{operation_name}") yield Tell(f"Completed: {operation_name}") return result @@ -235,7 +248,6 @@ def with_error_handling(operation, operation_name): @do def handle_error(operation_name, error): yield Tell(f"Error in {operation_name}: {error}") - yield Annotate({"error": str(error), "operation": operation_name}) # Usage @do @@ -251,6 +263,10 @@ def app(): Exponential backoff for retries: ```python +from doeff import Await, Try, do +from doeff_core_effects import Tell +import asyncio + @do def retry_with_exponential_backoff(operation, max_attempts=5): for attempt in range(1, max_attempts + 1): @@ -279,6 +295,9 @@ def fetch_with_retry(): Ensure cleanup with try/finally: ```python +from doeff import do +from doeff_core_effects import Ask, Tell + @do def with_resource(acquire, release, operation): resource = yield acquire() @@ -316,11 +335,13 @@ def use_database_connection(): Prevent cascading failures: ```python +from doeff import Try, do +from doeff_core_effects import Get, Put, Tell from doeff_time import GetTime @do def circuit_breaker(operation, threshold=5, timeout=60): - failures = yield AtomicGet("circuit_failures") + failures = yield Get("circuit_failures") last_failure_time = yield Get("circuit_last_failure") # Check if circuit is open @@ -331,7 +352,7 @@ def circuit_breaker(operation, threshold=5, timeout=60): raise Exception("Circuit breaker is open") else: # Reset after timeout - yield AtomicUpdate("circuit_failures", lambda _: 0) + yield Put("circuit_failures", 0) yield Put("circuit_last_failure", None) # Try operation @@ -339,11 +360,12 @@ def circuit_breaker(operation, threshold=5, timeout=60): if safe_result.is_ok(): # Success - reset counter - yield AtomicUpdate("circuit_failures", lambda _: 0) + yield Put("circuit_failures", 0) return safe_result.value else: # Failure - update circuit breaker state - yield AtomicUpdate("circuit_failures", lambda x: x + 1) + current_failures = yield Get("circuit_failures") + yield Put("circuit_failures", current_failures + 1) yield Put("circuit_last_failure", now) raise safe_result.error ``` @@ -355,6 +377,9 @@ def circuit_breaker(operation, threshold=5, timeout=60): Explicit state transitions: ```python +from doeff import do +from doeff_core_effects import Get, Put, Tell + @do def order_state_machine(order_id): state = yield Get(f"order_{order_id}_state") @@ -385,6 +410,8 @@ def order_state_machine(order_id): Functional state transformations: ```python +from doeff import do +from doeff_core_effects import Get, Put from doeff_time import GetTime @do @@ -412,6 +439,9 @@ def change_username(user_id, new_username): Save and restore state: ```python +from doeff import Try, do +from doeff_core_effects import Get, Put, Tell + @do def with_state_snapshot(operation): # Snapshot state @@ -436,26 +466,24 @@ def with_state_snapshot(operation): Fetch multiple resources concurrently: ```python +from doeff import Spawn, Gather, do from doeff_time import GetTime @do def fetch_user_dashboard(user_id): - # Fetch all data in parallel using Gather + dict reconstruction - programs = { - "user": fetch_user(user_id), - "posts": fetch_user_posts(user_id), - "followers": fetch_followers(user_id), - "notifications": fetch_notifications(user_id) - } - keys = list(programs.keys()) - values = yield Gather(*programs.values()) - results = dict(zip(keys, values)) - + # Spawn all fetches, then gather results + t_user = yield Spawn(fetch_user(user_id)) + t_posts = yield Spawn(fetch_user_posts(user_id)) + t_followers = yield Spawn(fetch_followers(user_id)) + t_notifications = yield Spawn(fetch_notifications(user_id)) + + results = yield Gather(t_user, t_posts, t_followers, t_notifications) + return { - "user": results["user"], - "posts": results["posts"], - "followers": results["followers"], - "notifications": results["notifications"], + "user": results[0], + "posts": results[1], + "followers": results[2], + "notifications": results[3], "dashboard_loaded_at": yield GetTime() } ``` @@ -465,6 +493,9 @@ def fetch_user_dashboard(user_id): Group operations for efficiency: ```python +from doeff import Spawn, Gather, do +from doeff_core_effects import Tell + @do def batch_processor(items, batch_size=100): results = [] @@ -473,10 +504,11 @@ def batch_processor(items, batch_size=100): batch = items[i:i + batch_size] yield Tell(f"Processing batch {i//batch_size + 1}") - # Use Gather to run Programs in parallel - batch_results = yield Gather(*[ - process_item(item) for item in batch - ]) + # Spawn tasks and gather results + tasks = [] + for item in batch: + tasks.append((yield Spawn(process_item(item)))) + batch_results = yield Gather(*tasks) results.extend(batch_results) @@ -490,48 +522,51 @@ def batch_processor(items, batch_size=100): Verify effects were executed: ```python +from doeff import do, run +from doeff_core_effects import Put, Tell +from doeff_core_effects.handlers import state, writer + def test_effects_executed(): @do def program(): yield Put("key", "value") yield Tell("Logged message") - yield Step("step1") return "result" - from doeff import run, default_handlers + # Run with handlers composed individually + result = run(writer(state(program()))) - # Run with result - result = run(program(), default_handlers()) - - # Verify result - assert result.is_ok() - assert result.value == "result" + # run() returns the raw value directly + assert result == "result" ``` ### Property-Based Testing ```python from hypothesis import given, strategies as st -from doeff import run, default_handlers +from doeff import do, run +from doeff_core_effects import Get, Put +from doeff_core_effects.handlers import state @given(st.integers(min_value=0, max_value=1000)) def test_counter_properties(initial_value): @do def counter_program(): yield Put("counter", initial_value) - yield Modify("counter", lambda x: x + 1) + current = yield Get("counter") + yield Put("counter", current + 1) result = yield Get("counter") return result - result = run(counter_program(), default_handlers()) + result = run(state(counter_program())) # Property: counter should always increment by 1 - assert result.value == initial_value + 1 + assert result == initial_value + 1 ``` ## Anti-Patterns -### ❌ Blocking Operations in Programs +### Bad: Blocking Operations in Programs **BAD:** ```python @@ -545,15 +580,17 @@ def bad_program(): **GOOD:** ```python +from doeff import Await, do +import asyncio + @do def good_program(): # DO: use Await with async - import asyncio yield Await(asyncio.sleep(5)) return "done" ``` -### ❌ Untracked Side Effects +### Bad: Untracked Side Effects **BAD:** ```python @@ -568,15 +605,13 @@ def bad_side_effects(): **GOOD:** ```python from dataclasses import dataclass -from doeff import EffectBase - +from doeff_vm import EffectBase @dataclass(frozen=True) class WriteFile(EffectBase): path: str data: str - @do def good_side_effects(): # Do: model the side effect explicitly @@ -584,7 +619,7 @@ def good_side_effects(): return "done" ``` -### ❌ Overusing State +### Bad: Overusing State **BAD:** ```python @@ -605,7 +640,7 @@ def good_parameter_passing(): return yield compute(arg1, arg2, arg3) ``` -### ❌ Ignoring Errors +### Bad: Ignoring Errors **BAD:** ```python @@ -617,20 +652,22 @@ def bad_error_handling(): **GOOD:** ```python +from doeff import Try, do +from doeff_core_effects import Tell + @do def good_error_handling(): result = yield Try(risky_operation()) if result.is_err(): yield Tell(f"Error occurred: {result.error}") - yield Annotate({"error": str(result.error)}) # Return default value or re-raise raise result.error return result.value ``` -### ❌ Deep Nesting +### Bad: Deep Nesting **BAD:** ```python diff --git a/docs/13-api-reference.md b/docs/13-api-reference.md index ad9a11c10..7ce730326 100644 --- a/docs/13-api-reference.md +++ b/docs/13-api-reference.md @@ -4,35 +4,38 @@ Complete API reference for doeff's current public surface. ## Core Types -### Program[T] +### Program (DoExpr) The core abstraction for lazy, reusable effectful computations. -```python -class Program[T]: - def map(self, f: Callable[[T], U]) -> Program[U] - def flat_map(self, f: Callable[[T], Program[U]]) -> Program[U] - def and_then_k(self, binder: Callable[[T], Program[U]]) -> Program[U] +`Program` is a virtual type alias for `DoExpr`. It is a union of all program node types +(`Pure`, `Expand`, `Perform`, `WithHandlerType`, `WithObserve`, etc.). There are no +`.map()`, `.flat_map()`, `.pure()`, or `.lift()` methods. Compose programs using the +`@do` decorator and `yield`. - @staticmethod - def pure(value: T) -> Program[T] +```python +from doeff import Program, DoExpr - @staticmethod - def lift(value: Program[U] | U) -> Program[U] +isinstance(my_node, Program) # True for any program node +isinstance(my_node, DoExpr) # equivalent ``` **See:** [Core Concepts](02-core-concepts.md#program-model) --- -### EffectBase / EffectValue[T] +### EffectBase Effects are user-space operation payloads. They are data, not execution control nodes. +All effects inherit from `EffectBase` (imported from `doeff_vm`). ```python -@dataclass(frozen=True, kw_only=True) -class EffectBase: - created_at: EffectCreationContext | None = None +from doeff_vm import EffectBase + +class MyEffect(EffectBase): + def __init__(self, payload): + super().__init__() + self.payload = payload ``` At runtime, effect dispatch happens through `Perform(effect)`. @@ -41,42 +44,9 @@ At runtime, effect dispatch happens through `Perform(effect)`. --- -### RunResult[T] - -Result container returned by `run()` and `async_run()`. - -```python -class RunResult[T](Protocol): - @property - def result(self) -> Result[T]: ... - - @property - def raw_store(self) -> dict[str, Any]: ... - - @property - def value(self) -> T: ... - - @property - def error(self) -> BaseException: ... - - def is_ok(self) -> bool: ... - def is_err(self) -> bool: ... -``` - -**Core fields:** - -- **`result`** - `Ok(value)` or `Err(error)` -- **`raw_store`** - Final store snapshot -- **`value`** - Unwraps `Ok` value (raises on `Err`) -- **`error`** - Unwraps `Err` error (raises on `Ok`) - -**See:** [Error Handling](05-error-handling.md#runresult-overview) - ---- - ### Result[T] -Success/failure sum type used by `RunResult.result` and `Try(...)`. +Success/failure sum type used by `Try(...)`. ```python Result[T] = Ok[T] | Err @@ -118,65 +88,75 @@ Failure constructor for `Result[T]`. err = Err(ValueError("invalid input")) ``` -**Signature:** -`Err(error: Exception, captured_traceback: Maybe[EffectTraceback] = NOTHING)` +**Signature:** `Err(error: Exception)` **Fields:** - **`error`** - The captured exception -- **`captured_traceback`** - Optional effect traceback metadata. Defaults to `NOTHING`, - and is populated when traceback capture is enabled for that error path. **Pattern matching:** ```python match result: - case Err(error=e, captured_traceback=tb): - print("failed", e, tb) + case Err(error=e): + print("failed", e) ``` --- -## Decorators +### Maybe -### @do - -Converts a generator function into a `KleisliProgram`. +Optional value type. ```python -@do -def my_program(x: int) -> Program[int]: - value = yield Get("key") - return value + x -``` +from doeff import Some, Nothing -**Returns:** `KleisliProgram[..., T]` +maybe_val = Some(42) +empty = Nothing -**See:** [Core Concepts](02-core-concepts.md#generator-as-ast), -[Kleisli Arrows](11-kleisli-arrows.md) +match maybe_val: + case Some(value=v): + print("has value", v) + case Nothing: + print("empty") +``` + +`Maybe = Some | Nothing` --- -### @cache +## Decorators + +### @do -Caches Program results with policy fields. +Converts a generator function into a callable that produces `Expand` objects (program nodes). ```python -@cache(ttl=60, lifecycle=CacheLifecycle.SESSION) +from doeff import do + @do -def expensive_computation(x: int): - return x * 2 +def my_program(x: int): + value = yield Get("key") + return value + x + +prog = my_program(42) # returns Expand (a DoExpr node), not executed yet +result = run(prog) # execute ``` -**Primary parameters:** +**Returns:** `Callable[..., Expand]` -- **`ttl`** (`float | None`) - Time-to-live in seconds -- **`lifecycle`** (`CacheLifecycle | str | None`) - Cache lifecycle hint -- **`storage`** (`CacheStorage | str | None`) - Storage hint -- **`metadata`** (`Mapping[str, Any] | None`) - Policy metadata -- **`policy`** (`CachePolicy | Mapping[str, Any] | None`) - Full policy object +Accepts an optional `non_tail` keyword argument to suppress warnings about non-tail +`Resume`/`ResumeThrow` usage in handler functions: -**See:** [Cache System](07-cache-system.md), [cache.md](cache.md) +```python +@do(non_tail=True) +def my_handler(effect, k): + value = yield Resume(k, 42) + # ... use value (non-tail position) ... + return value +``` + +**See:** [Core Concepts](02-core-concepts.md#generator-as-ast) --- @@ -186,11 +166,10 @@ def expensive_computation(x: int): Request an environment value. -Raises `MissingEnvKeyError` (a `KeyError` subtype) when `key` is not present in the current -environment. +Raises `KeyError` when `key` is not present in the current environment. If the environment value is a `Program`, `Ask` evaluates it lazily once per `run()` invocation and -caches the computed value per key. Concurrent `Ask` calls for the same lazy key are coordinated so +caches the computed value per key (when using `lazy_ask`). Concurrent `Ask` calls for the same lazy key are coordinated so only one task performs evaluation while others wait cooperatively. `Local(...)` overrides can invalidate that per-run cache for overridden keys. @@ -198,13 +177,13 @@ invalidate that per-run cache for overridden keys. config = yield Ask("database_url") ``` -**Signature:** `Ask(key: EnvKey)` +**Signature:** `Ask(key)` **See:** [Basic Effects](03-basic-effects.md#reader-effects) --- -### Local(env_update, sub_program) +### Local(env, program) Run a sub-program with environment overrides. @@ -212,8 +191,7 @@ Run a sub-program with environment overrides. result = yield Local({"timeout": 30}, sub_program()) ``` -**Signature:** -`Local(env_update: Mapping[Any, object], sub_program: ProgramLike)` +**Signature:** `Local(env: Mapping, program: DoExpr)` **See:** [Basic Effects](03-basic-effects.md#local) @@ -229,9 +207,9 @@ Read state value. count = yield Get("counter") ``` -**Signature:** `Get(key: str)` +**Signature:** `Get(key)` -Raises `KeyError` when `key` is missing. +Returns `None` when `key` is missing (with the default `state()` handler). --- @@ -243,121 +221,224 @@ Write state value. yield Put("counter", 42) ``` -**Signature:** `Put(key: str, value: Any)` +**Signature:** `Put(key, value)` + +**See:** [Basic Effects](03-basic-effects.md#state-effects) --- -### Modify(key, f) +## Writer Effects + +### Tell(message) -Update state value using a transformation function. +Append a log entry. `Tell(message)` is a convenience function that creates a +`WriterTellEffect(message)`. ```python -yield Modify("counter", lambda x: x + 1) +yield Tell("Processing started") ``` -**Signature:** `Modify(key: str, f: Callable[[Any | None], Any])` +**Signature:** `Tell(message) -> WriterTellEffect` + +--- -If `key` is missing, `Modify` calls `f(None)` (it does not raise `KeyError`). +### slog(msg, **kwargs) -`Modify` is atomic: if `f` raises, the store is left unchanged. +Append a structured log entry. Creates a `WriterTellEffect` with keyword arguments. -**See:** [Basic Effects](03-basic-effects.md#state-effects) +```python +yield slog("Processing", count=42, level="info") +``` + +**Signature:** `slog(msg, **kwargs) -> WriterTellEffect` + +`Slog` is an alias for `WriterTellEffect`. --- -## Writer Effects +### WriterTellEffect(msg, **kwargs) -### Tell(message) +The underlying effect type for both `Tell` and `slog`. `Listen` collects these. + +```python +yield WriterTellEffect("event", severity="warn") +``` -Append a log entry. +**Signature:** `WriterTellEffect(msg, **kwargs)` + +**Fields:** + +- **`msg`** - The log message +- **`kwargs`** - Additional structured key-value pairs + +--- + +### Listen(program, types=None) + +Run a sub-program and collect all effects of the given types emitted during execution. ```python -yield Tell("Processing started") +value, collected = yield Listen(sub_program()) +``` + +**Signature:** `Listen(program, types=None)` + +**Returns:** `(value, list)` tuple -- the program result and a list of collected effects. + +When `types` is `None`, defaults to collecting `WriterTellEffect` instances. + +**See:** [Basic Effects](03-basic-effects.md#writer-effects) + +--- + +## Error Handling Effects + +### Try(program) + +Run a sub-program and capture errors as `Result`. + +```python +result = yield Try(risky_operation()) +match result: + case Ok(value=v): + return v + case Err(error=e): + return "fallback" ``` -**Signature:** `Tell(message: object)` +**Signature:** `Try(program: DoExpr)` + +**Returns:** `Ok(value)` or `Err(error)` -`Log` is a deprecated alias for `Tell`. Use `Tell` instead. +**See:** [Error Handling](05-error-handling.md#try-effect) --- -### Listen(sub_program) +## Control Effects + +### Pure(value) -Run a sub-program and capture emitted writer logs. +Wrap a plain value as a DoExpr program node. ```python -result = yield Listen(sub_program()) -value, logs = result +value = yield Pure({"status": "ok"}) ``` -**Signature:** `Listen(sub_program: ProgramLike)` +**Signature:** `Pure(value)` + +--- -**Returns:** `ListenResult(value: T, log: BoundedLog)` +### Pass(effect, k) -`ListenResult.log` uses bounded retention semantics: when capacity is exceeded, oldest -entries are evicted. +Forward an unhandled effect to the next outer handler. Used inside handler functions +to indicate "I don't handle this effect". + +```python +@do +def my_handler(effect, k): + if isinstance(effect, MyEffect): + result = yield Resume(k, effect.payload * 2) + return result + yield Pass(effect, k) # forward everything else +``` + +**Signature:** `Pass(effect, k)` --- -### StructuredLog(**entries) +### WithObserve(observer, body) -Append a structured log payload. +Install a cross-cutting observer and run a body program under it. The observer is +called for each effect performed during execution. ```python -yield StructuredLog(level="info", message="Processing", count=42) +def my_observer(effect): + print(f"observed: {effect}") + +result = yield WithObserve(my_observer, sub_program()) ``` -**Signature:** `StructuredLog(**entries: object)` +**Signature:** `WithObserve(observer: callable, body: DoExpr)` -**See:** [Basic Effects](03-basic-effects.md#writer-effects) +Accepts a plain Python callable as observer -- automatically wraps it with `doeff_vm.Callable` +so the Rust VM can invoke it. + +--- + +### Handler Installer Call + +Run a scoped sub-program under a custom handler by calling a `Program -> Program` +handler installer. + +```python +from doeff_core_effects.handlers import reader + +scoped = reader(env={"name": "override"})(worker()) +``` + +**Shape:** `handler(program: DoExpr) -> DoExpr` + +Handlers built with `doeff.program.handler()` have this `Program -> Program` shape. +Compose them by nesting calls: + +```python +prog = my_program() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={"key": "value"})(prog) +result = run(scheduled(prog)) +``` --- ## Async and Concurrency Effects -### Await(awaitable) +### Await(coroutine) -Await a Python awaitable. +Await a Python coroutine or future. Bridges async into doeff. ```python value = yield Await(async_call()) ``` -**Signature:** `Await(awaitable: Awaitable[Any])` +**Signature:** `Await(coroutine)` -`Await` bridges Python `asyncio` awaitables into doeff. For doeff-native `Task`/`Future` -handles, use `Wait` instead. +Requires `await_handler()` and `scheduled()` to be installed. --- -### Spawn(program, **options) +### Spawn(program, priority=PRIORITY_NORMAL) -Spawn a Program in the background and return a `Task` handle. +Spawn a program in the background and return a `Task` handle. ```python task = yield Spawn(worker()) result = yield Wait(task) ``` +**Signature:** `Spawn(program, priority=PRIORITY_NORMAL)` + +**Returns:** `Task` + **See:** [Advanced Effects](09-advanced-effects.md) --- -### Wait(future) +### Wait(task) -Wait for a `Task`/`Future` waitable value. +Wait for a `Task` or `Future` handle to complete. ```python result = yield Wait(task) ``` -**Signature:** `Wait(future: Waitable[T])` +**Signature:** `Wait(task, priority=None)` --- -### Gather(*items) +### Gather(*tasks) -Resolve multiple waitables and return results in input order. +Wait for multiple tasks/futures and return results in input order. ```python task_1 = yield Spawn(fetch_user(1)) @@ -366,51 +447,39 @@ task_3 = yield Spawn(fetch_user(3)) results = yield Gather(task_1, task_2, task_3) ``` -**Signature:** `Gather(*items: Waitable[Any])` +**Signature:** `Gather(*tasks)` **See:** [Advanced Effects](09-advanced-effects.md#gather-effects) --- -### Race(*waitables) +### Race(*tasks) -Wait for the first waitable to complete. +Wait for the first task/future to complete. ```python winner = yield Race(task_a, task_b) -value = winner.value ``` -**Signature:** `Race(*futures: Waitable[Any])` - -**Returns:** `RaceResult(first, value, rest)` +**Signature:** `Race(*tasks)` **See:** [Async Effects](04-async-effects.md#race-semantics), [Advanced Effects](09-advanced-effects.md#race-effect) --- -### Task.cancel() +### Cancel(task) Request cooperative cancellation for a target `Task`. ```python task = yield Spawn(worker()) -_ = yield task.cancel() +yield Cancel(task) ``` -**Signature:** `Task.cancel()` - -`Task.cancel()` is effectful: it returns a cancellation effect that must be yielded. +**Signature:** `Cancel(task)` -Cancellation behavior depends on the target task state: - -- **`Pending`** - Mark cancelled immediately and wake waiters with `TaskCancelledError`. -- **`Running`** - Set cooperative cancel flag; task is cancelled at next scheduler yield point. -- **`Suspended`** - Mark cancelled immediately and wake waiters with `TaskCancelledError`. -- **`Blocked`** - Mark cancelled, remove target wait registration, wake waiters with - `TaskCancelledError`. -- **`Completed` / `Failed` / `Cancelled`** - No-op. +`Cancel` is an effect that must be yielded. There is no `task.cancel()` method. **See:** [Async Effects](04-async-effects.md#cancel-and-taskcancellederror) @@ -422,13 +491,72 @@ Raised by `Wait`, `Gather`, or `Race` when a waited task was cancelled. ```python joined = yield Try(Wait(task)) -if joined.is_err() and joined.error.__class__.__name__ == "TaskCancelledError": - ... +match joined: + case Err(error=e) if isinstance(e, TaskCancelledError): + ... # handle cancellation ``` -**Signature:** `class TaskCancelledError(Exception)` +**Import:** `from doeff import TaskCancelledError` + +--- + +## Promise Effects + +### CreatePromise() + +Create a scheduler-internal promise and return a `Promise` handle. + +```python +promise = yield CreatePromise() +future = promise.future # read-side Future handle +``` + +**Signature:** `CreatePromise()` + +**Returns:** `Promise` (with `.future` property for `Wait`) + +--- + +### CompletePromise(promise, value) + +Complete a promise with a success value. + +```python +yield CompletePromise(promise, result_value) +``` -**Import:** `from doeff.effects import TaskCancelledError` +**Signature:** `CompletePromise(promise, value)` + +--- + +### FailPromise(promise, error) + +Complete a promise with an error. + +```python +yield FailPromise(promise, ValueError("something went wrong")) +``` + +**Signature:** `FailPromise(promise, error)` + +--- + +### CreateExternalPromise() + +Create a thread-safe external promise. The returned `ExternalPromise` has `.complete(value)` +and `.fail(error)` methods that can be called from any thread. + +```python +ep = yield CreateExternalPromise() +# From another thread: +ep.complete(result) +# or ep.fail(error) +value = yield Wait(ep.future) +``` + +**Signature:** `CreateExternalPromise()` + +**Returns:** `ExternalPromise` (with `.future` property and thread-safe `.complete()`/`.fail()`) --- @@ -438,23 +566,17 @@ if joined.is_err() and joined.error.__class__.__name__ == "TaskCancelledError": Create an opaque semaphore handle with `permits` initial permits. -`permits` must be `>= 1`; `CreateSemaphore(0)` raises -`ValueError("permits must be >= 1")`. - ```python sem = yield CreateSemaphore(3) ``` -**Signature:** `CreateSemaphore(permits: int)` +**Signature:** `CreateSemaphore(permits=1)` **Returns:** `Semaphore` -The returned `Semaphore` is opaque. Keep and reuse the handle returned by `CreateSemaphore`. -`Semaphore(...)` is not part of the public API and raises `TypeError`. - --- -### AcquireSemaphore(sem) +### AcquireSemaphore(semaphore) Acquire one permit from a semaphore; blocks cooperatively when no permits are available. @@ -466,22 +588,15 @@ finally: yield ReleaseSemaphore(sem) ``` -**Signature:** `AcquireSemaphore(semaphore: Semaphore)` - -`semaphore` must be the handle returned by `CreateSemaphore` or another Python reference to that -same handle object. `Semaphore(id)` construction is not supported. +**Signature:** `AcquireSemaphore(semaphore)` -Blocked acquirers are resumed in FIFO order. When no permit is available, the task transitions to -`BLOCKED` until a release occurs. - -If a blocked waiter is cancelled, it is removed from the semaphore queue, raises -`TaskCancelledError`, and consumes no permit. +Blocked acquirers are resumed in FIFO order. **See:** [Semaphore Effects](21-semaphore-effects.md#acquiresemaphoresem) --- -### ReleaseSemaphore(sem) +### ReleaseSemaphore(semaphore) Release one permit back to a semaphore. @@ -489,325 +604,333 @@ Release one permit back to a semaphore. yield ReleaseSemaphore(sem) ``` -**Signature:** `ReleaseSemaphore(semaphore: Semaphore)` - -`semaphore` must be the runtime-created handle returned by `CreateSemaphore` (or another Python -reference to that same handle object). `Semaphore(id)` construction is not supported. +**Signature:** `ReleaseSemaphore(semaphore)` -When waiters exist, release uses direct handoff to the oldest waiter (FIFO): the permit transfers -to that waiter and `available_permits` remains `0`. +When waiters exist, release uses direct handoff to the oldest waiter (FIFO). -Permit leak warning: semaphores do not track ownership. If a task fails or is cancelled after -acquire and before release, that permit is leaked. Always guard critical sections with -`try/finally` so `ReleaseSemaphore` still runs. - -Lifecycle: there is no explicit destroy API; semaphore state is released when handles are garbage -collected. +Always guard critical sections with `try/finally` so `ReleaseSemaphore` still runs +if the task fails or is cancelled. **See:** [Semaphore Effects](21-semaphore-effects.md#releasesemaphoresem) --- -## Control Effects +## Cache Effects -### Pure(value) +Located in `doeff_core_effects.cache_effects`. -Return an immediate value without mutating state, environment, or writer log. +### CacheGet(key) -On the control path, `yield Pure(x)` is equivalent to returning `x` directly. +Retrieve a cached value. ```python -value = yield Pure({"status": "ok"}) +from doeff_core_effects.cache_effects import CacheGet + +value = yield CacheGet("expensive_key") ``` -**Signature:** `Pure(value: Any)` +**Signature:** `CacheGet(key) -> CacheGetEffect` --- -### Handler Installer Call +### CachePut(key, value, ttl=None, *, lifecycle=None, storage=None, metadata=None, policy=None) -Run a scoped sub-program under a custom handler by calling a Program -> Program handler installer. +Store a value in cache with optional policy. ```python -from doeff_core_effects.handlers import reader +from doeff_core_effects.cache_effects import CachePut +from doeff_core_effects.cache_policy import CacheLifecycle -scoped = reader(env={"name": "override"})(worker()) +yield CachePut( + "key", + value, + ttl=300, + lifecycle=CacheLifecycle.PERSISTENT, +) ``` -**Shape:** `handler(program: Program[T]) -> Program[T]` +**Signature:** `CachePut(key, value, ttl=None, *, lifecycle=None, storage=None, metadata=None, policy=None) -> CachePutEffect` -**Notes:** +--- -- New-style handlers built with `defhandler` already have this Program -> Program shape. -- Python handler factories should return this shape rather than asking callers to use the - deprecated `WithHandler` compatibility shim. -- The raw handler's first-parameter effect annotation is converted into a runtime type filter - inside the installed handler scope. -- Use `yield Pass()` for transparent fallthrough. -- Use `yield Delegate()` only when the handler intentionally wants the outer handler's result. +### CachePolicy -### Deprecated WithHandler Shim +```python +from doeff_core_effects.cache_policy import CachePolicy, CacheLifecycle, CacheStorage +``` -`WithHandler` remains exported for backward compatibility. New code should call a handler installer -directly, for example `handler(program)`. `WithHandlerType` is the low-level VM node/type alias for -code that needs direct DoExpr construction or `isinstance` checks. +**CacheLifecycle** enum: `TRANSIENT`, `SESSION`, `PERSISTENT` ---- +**CacheStorage** enum: `MEMORY`, `DISK` -### WithIntercept(f, expr, types=None, mode="include") +--- -Run a scoped program under interception. Each matched yielded value is offered to `f`. +## Memo Effects -```python -@do -def transform(effect): - if isinstance(effect, AskEffect) and effect.key == "timeout": - return Pure(30) - return effect +Located in `doeff_core_effects.memo_effects`. Memo effects are the cost-aware memoization +system that replaced the old `CacheGet` name in the `doeff` top-level namespace. -result = yield WithIntercept(transform, worker(), types=(AskEffect,), mode="include") -``` +### MemoGet(key) -**Signature:** -`WithIntercept(f: HandlerFn, expr: DoExpr[T], types: Iterable[type] | None = None, mode: str = "include")` +Retrieve a memoized value. -**Interceptor contract:** +```python +from doeff_core_effects.memo_effects import MemoGet -- Return the original effect to pass through unchanged. -- Return an `Effect` or `Program` to replace that step. -- Interceptor errors propagate. +value = yield MemoGet("computation_result") +``` -Interception propagates to child execution contexts (for example `Gather`, `Try`, and `Spawn`). +**Signature:** `MemoGet(key, *, recompute_cost=RecomputeCost.CHEAP) -> MemoGetEffect` --- -## Error Handling Effects +### MemoPut(key, value, ...) -### Try(sub_program) - -Run a sub-program and capture errors as `Result`. +Store a memoized value. ```python -result = yield Try(risky_operation()) -if result.is_ok(): - return result.value -return "fallback" -``` +from doeff_core_effects.memo_effects import MemoPut -**Signature:** `Try(sub_program: ProgramLike)` +yield MemoPut("key", value, recompute_cost="expensive") +``` -**See:** [Error Handling](05-error-handling.md#try-effect) +**Signature:** `MemoPut(key, value, ttl=None, *, recompute_cost=None, lifecycle=None, metadata=None, policy=None, source_effect=None) -> MemoPutEffect` --- -## Cache Effects +## Execution -### CacheGet(key) +### run(doexpr) -Retrieve a cached value. +Execute a DoExpr program to completion and return the raw result value. ```python -value = yield CacheGet("expensive_key") -``` - -**Signature:** `CacheGet(key: Any)` - ---- +from doeff import do, run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled -### CachePut(key, value, ttl=None, *, lifecycle=None, storage=None, metadata=None, policy=None) +@do +def my_program(): + name = yield Ask("name") + yield Put("greeted", True) + yield Tell(f"Hello, {name}") + return f"Done greeting {name}" + +prog = my_program() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={"name": "Alice"})(prog) +result = run(scheduled(prog)) +# result is "Done greeting Alice" (raw value, not a wrapper) +``` -Store a value in cache. +**Signature:** ```python -yield CachePut( - "key", - value, - ttl=300, - lifecycle=CacheLifecycle.PERSISTENT, -) +def run(doexpr) -> T ``` -**See:** [Cache System](07-cache-system.md), [cache.md](cache.md) +`run()` takes a single argument -- the program to execute. There are no `handlers`, `env`, +or `store` keyword arguments. Compose handlers by wrapping the program before passing it +to `run()`. Use `scheduled()` as the outermost wrapper when using scheduler effects +(`Spawn`, `Wait`, `Gather`, `Race`, semaphores, promises). ---- +On error, exceptions propagate with enriched doeff traceback information printed to stderr. -## Graph Effects +--- -### Step(value, meta=None) +### scheduled(program) -Add a step node to the execution graph. +Wrap a program with the cooperative scheduler. Returns a new DoExpr that, when passed to +`run()`, enables all scheduler effects (Spawn, Wait, Gather, Race, Cancel, promises, +semaphores). ```python -yield Step("initialize", {"phase": "setup"}) +from doeff_core_effects.scheduler import scheduled + +result = run(scheduled(prog)) ``` -**Signature:** `Step(value: Any, meta: dict[str, Any] | None = None)` +**Signature:** `scheduled(program) -> DoExpr` + +This is the replacement for the deleted `async_run()`. All async/concurrent effects +require `scheduled()` to be installed. --- -### Annotate(meta) +## Handlers -Add metadata to the latest graph step. +Handlers are `Program -> Program` functions. Compose them by calling them in sequence: ```python -yield Annotate({"user_id": 123, "operation": "fetch"}) +from doeff import do, run +from doeff_core_effects.handlers import reader, state, writer, try_handler, slog_handler +from doeff_core_effects.scheduler import scheduled + +prog = my_program() +prog = slog_handler(prog) # innermost +prog = writer(prog) +prog = state(initial={"k": 0})(prog) +prog = reader(env={"key": "val"})(prog) # outermost user handler +result = run(scheduled(prog)) # scheduler wraps everything ``` -**Signature:** `Annotate(meta: dict[str, Any])` - ---- - -### Snapshot() +### reader(env=None) -Capture current graph state. +Handles `Ask(key)` by looking up `key` in `env`. Raises `KeyError` on miss. ```python -graph = yield Snapshot() +from doeff_core_effects.handlers import reader + +prog = reader(env={"db_url": "postgres://..."})(my_program()) ``` --- -### CaptureGraph(program) +### lazy_ask(env=None, *, strict=False) -Run a sub-program and capture its graph output. +Handles `Ask` and `Local` with lazy evaluation and caching. If an env value is a +`Program`, evaluates it lazily once per key with concurrent coordination via semaphores. ```python -value, graph = yield CaptureGraph(sub_program()) +from doeff_core_effects.handlers import lazy_ask + +prog = lazy_ask(env={"config": load_config_program()})(my_program()) ``` -**Signature:** `CaptureGraph(program: ProgramLike)` +- `strict=False` (default): unresolved keys are forwarded to outer handlers via `Pass`. +- `strict=True`: unresolved keys raise `KeyError`. -**See:** [Graph Tracking](08-graph-tracking.md#capturegraph) +Requires `scheduled()` to be installed (for semaphore support). --- -## Atomic Effects +### state(initial=None) -### AtomicGet(key, *, default_factory=None) - -Thread-safe shared-state read. +Handles `Get(key)` and `Put(key, value)` with a mutable dict. ```python -count = yield AtomicGet("counter") -``` +from doeff_core_effects.handlers import state -**Signature:** -`AtomicGet(key: str, *, default_factory: Callable[[], Any] | None = None)` +prog = state(initial={"counter": 0})(my_program()) +``` --- -### AtomicUpdate(key, updater, *, default_factory=None) +### writer -Thread-safe shared-state update. +Handles `Tell(message)` / `WriterTellEffect`. Collects messages into a list. +Pre-installed `Program -> Program` handler (not a factory -- use directly). +Requires `state()` to be installed as an outer handler. + +Access the log from inside a `@do` program via `yield writer_log()`. ```python -new_value = yield AtomicUpdate("counter", lambda x: x + 1) -``` +from doeff_core_effects.handlers import writer, writer_log -**Signature:** -`AtomicUpdate(key: str, updater: Callable[[Any], Any], *, default_factory: Callable[[], Any] | None = None)` +prog = writer(my_program()) +prog = state()(prog) +run(scheduled(prog)) + +# To access the log, use writer_log() inside a @do program: +@do +def with_log_access(): + yield my_program() + log = yield writer_log() + return log +``` --- -## Execution +### slog_handler -### run +Handles `Slog` / `WriterTellEffect` structured log entries. Collects entries as dicts +with `msg` and all kwargs. Pre-installed `Program -> Program` handler (not a factory -- +use directly). Requires `state()` to be installed as an outer handler. -Synchronously execute a Program/effect with explicit handler stack. +Access the log from inside a `@do` program via `yield slog_log()`. ```python -from doeff import default_handlers, run +from doeff_core_effects.handlers import slog_handler, slog_log -result = run( - my_program(), - handlers=default_handlers(), - env={"key": "value"}, - store={"state": 0}, -) +prog = slog_handler(my_program()) +prog = state()(prog) +run(scheduled(prog)) + +# To access the log, use slog_log() inside a @do program: +@do +def with_slog_access(): + yield my_program() + log = yield slog_log() + return log +# log is [{"msg": "event", "count": 42}, ...] ``` -**Signature:** +--- + +### try_handler + +Handles `Try(program)` -- wraps inner program execution and catches errors as +`Ok(value)` or `Err(error)`. Pre-installed (not a factory -- use directly). ```python -def run( - program: DoExpr[T] | EffectValue[T], - handlers: Sequence[Any] = (), - env: dict[Any, Any] | None = None, - store: dict[str, Any] | None = None, - trace: bool = False, -) -> RunResult[T] -``` +from doeff_core_effects.handlers import try_handler -`env` and `store` are the execution inputs (Reader and State roots). -`handlers` is a low-level runner hook. For custom handler composition, prefer direct -`handler(program)` calls. +prog = try_handler(my_program()) +``` --- -### async_run +### local_handler -Asynchronously execute a Program/effect with async-aware await handling. +Handles `Local(env, program)` -- scoped environment overrides. Pre-installed +(not a factory -- use directly). ```python -from doeff import async_run, default_async_handlers +from doeff_core_effects.handlers import local_handler -result = await async_run( - my_program(), - handlers=default_async_handlers(), - env={"key": "value"}, - store={"state": 0}, -) +prog = local_handler(my_program()) ``` -**Signature:** +--- -```python -async def async_run( - program: DoExpr[T] | EffectValue[T], - handlers: Sequence[Any] = (), - env: dict[Any, Any] | None = None, - store: dict[str, Any] | None = None, - trace: bool = False, -) -> RunResult[T] -``` +### listen_handler -`handlers` is a low-level runner hook. For custom handler composition, prefer direct -`handler(program)` calls. +Handles `Listen(program, types=...)` -- collects effects during sub-program execution. +Pre-installed (not a factory -- use directly). ---- +```python +from doeff_core_effects.handlers import listen_handler ---- +prog = listen_handler(my_program()) +``` -## Utilities +--- -### graph_to_html(graph, *, title="doeff Graph Snapshot", mark_success=False) +### await_handler() -Generate graph-visualization HTML as a Program. +Handles `Await(coroutine)` by bridging async into the scheduler via `ExternalPromise`. +Requires `scheduled()` to be installed. ```python -from doeff import default_handlers, graph_to_html, run +from doeff_core_effects.handlers import await_handler -html = run(graph_to_html(graph), handlers=default_handlers()).value +prog = await_handler()(my_program()) ``` -**Signature:** -`graph_to_html(graph: WGraph, *, title: str = ..., mark_success: bool = False) -> Program[str]` - --- -### write_graph_html(graph, output_path, *, title="doeff Graph Snapshot", mark_success=False) +### env_var_ask(*, prefix="DOEFF_") -Write graph HTML to a file as a Program. +Handles `Ask(key)` by looking up `os.environ[prefix + key]`. Supports `{module.path}` +syntax for importing symbols. Unresolved keys are forwarded to outer handlers. ```python -from doeff import default_handlers, run, write_graph_html +from doeff_core_effects.handlers import env_var_ask -path = run(write_graph_html(graph, "output.html"), handlers=default_handlers()).value +prog = env_var_ask()(my_program()) ``` -**Signature:** -`write_graph_html(graph: WGraph, output_path: str | Path, *, title: str = ..., mark_success: bool = False) -> Program[Path]` - --- ## Quick Reference @@ -816,54 +939,111 @@ path = run(write_graph_html(graph, "output.html"), handlers=default_handlers()). | Category | Effects | |----------|---------| -| **Reader** | Ask, Local | -| **State** | Get, Put, Modify | -| **Writer** | Tell, Listen, StructuredLog, slog | -| **Async/Concurrency** | Await, Spawn, Wait, Gather, Race, Task.cancel, TaskCancelledError | -| **Semaphore** | CreateSemaphore, AcquireSemaphore, ReleaseSemaphore | -| **Control** | Pure, handler installer calls, WithHandlerType, WithIntercept | -| **Error** | Try, Ok, Err | -| **Cache** | CacheGet, CachePut | -| **Graph** | Step, Annotate, Snapshot, CaptureGraph | -| **Atomic** | AtomicGet, AtomicUpdate | +| **Reader** | `Ask`, `Local` | +| **State** | `Get`, `Put` | +| **Writer** | `Tell`, `slog`, `WriterTellEffect`, `Listen` | +| **Async/Concurrency** | `Await`, `Spawn`, `Wait`, `Gather`, `Race`, `Cancel` | +| **Semaphore** | `CreateSemaphore`, `AcquireSemaphore`, `ReleaseSemaphore` | +| **Promise** | `CreatePromise`, `CompletePromise`, `FailPromise`, `CreateExternalPromise` | +| **Control** | `Pure`, `Pass`, `WithObserve`, handler installer calls | +| **Error** | `Try`, `Ok`, `Err` | +| **Cache** | `CacheGet`, `CachePut` (in `doeff_core_effects.cache_effects`) | +| **Memo** | `MemoGet`, `MemoPut` (in `doeff_core_effects.memo_effects`) | ### Common Imports ```python # Core -from doeff import Program, EffectBase, do +from doeff import Program, DoExpr, do # Execution -from doeff import run, async_run -from doeff import default_handlers, default_async_handlers +from doeff import run +from doeff_core_effects.scheduler import scheduled # Reader / State / Writer -from doeff import Ask, Local, Get, Put, Modify, Tell, Listen, StructuredLog, slog +from doeff import Ask, Local, Get, Put, Tell, Listen, slog, WriterTellEffect # Async / Concurrency -from doeff import Await, Spawn, Wait, Gather, Race, Task -from doeff.effects import TaskCancelledError # raised on Wait/Gather/Race for cancelled tasks +from doeff import Await, Spawn, Wait, Gather, Race, Cancel, Task +from doeff import TaskCancelledError # Semaphore from doeff import AcquireSemaphore, CreateSemaphore, ReleaseSemaphore, Semaphore +# Promise +from doeff import CreatePromise, CompletePromise, FailPromise, CreateExternalPromise + # Control -from doeff import Pure, WithHandlerType, WithIntercept +from doeff import Pure, Pass, WithObserve # Error handling from doeff import Try, Ok, Err -# Cache -from doeff import CacheGet, CachePut, cache +# Maybe +from doeff import Some, Nothing + +# Handlers +from doeff_core_effects.handlers import ( + reader, lazy_ask, state, writer, try_handler, + slog_handler, local_handler, listen_handler, await_handler, +) + +# Cache effects (separate package) +from doeff_core_effects.cache_effects import CacheGet, CachePut +from doeff_core_effects.cache_policy import CacheLifecycle, CachePolicy, CacheStorage + +# Memo effects (separate package) +from doeff_core_effects.memo_effects import MemoGet, MemoPut +``` -# Graph -from doeff import Step, Annotate, Snapshot, CaptureGraph, graph_to_html, write_graph_html +### Handler Composition Pattern -# Atomic -from doeff import AtomicGet, AtomicUpdate +```python +from doeff import do, run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled +@do +def my_program(): + name = yield Ask("name") + yield Put("count", 1) + yield Tell(f"greeted {name}") + return name + +prog = my_program() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={"name": "Alice"})(prog) +result = run(scheduled(prog)) ``` +--- + +## Removed APIs + +The following APIs have been removed. Using them raises `RuntimeError` with a migration hint. + +| Removed | Replacement | +|---------|-------------| +| `default_handlers` | Compose handlers by calling `handler(program)` | +| `async_run` | Use `run()` with `scheduled()` | +| `default_async_handlers` | Compose handlers by calling `handler(program)` | +| `RunResult` | `run()` returns raw values directly | +| `Modify` | Use `Get` + `Put` instead | +| `WithIntercept` | Use `WithObserve` instead | +| `KleisliProgram` | Use `@do` instead | +| `Delegate` | Use `yield effect` to re-perform in handler body | +| `cache` (decorator) | Cache module removed | +| `CacheGet` (top-level) | Renamed to `MemoGet` in `doeff_core_effects.memo_effects` | +| `graph_snapshot` | Concept removed | +| `Step`, `Annotate`, `Snapshot`, `CaptureGraph` | Graph tracking removed | +| `graph_to_html`, `write_graph_html` | Graph visualization removed | +| `AtomicGet`, `AtomicUpdate` | Concept removed | +| `AllocVar`, `ReadVar` | Use var_store directly | +| `presets` | Presets module removed | + +--- + ## Next Steps - **[Getting Started](01-getting-started.md)** - Begin using doeff diff --git a/docs/14-cli-auto-discovery.md b/docs/14-cli-auto-discovery.md index 865f3002b..30168835c 100644 --- a/docs/14-cli-auto-discovery.md +++ b/docs/14-cli-auto-discovery.md @@ -71,14 +71,16 @@ The CLI automatically: ```python # myapp/interpreter.py -from doeff import Program, default_handlers, run +from doeff import do, run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled def my_interpreter(prog: Program) -> any: """ Execute programs for myapp. # doeff: interpreter, default """ - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) ``` **Step 2**: Mark your environment (optional) @@ -196,7 +198,7 @@ The CLI searches for functions marked with `# doeff: interpreter, default`: # myapp/__init__.py def base_interpreter(prog: Program): """# doeff: interpreter, default""" - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) # myapp/features/__init__.py # (no interpreter) @@ -243,7 +245,7 @@ Interpreters must: ```python def my_interpreter(prog: Program): """# doeff: interpreter, default""" - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) ``` **Invalid** (async not allowed): @@ -352,13 +354,13 @@ def my_interpreter(prog: Program): Execute programs with custom settings. # doeff: interpreter, default """ - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) ``` 2. **Inline comment**: ```python def my_interpreter(prog: Program): # doeff: interpreter, default - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) ``` 3. **Multi-line signature**: @@ -497,7 +499,7 @@ myapp/ # myapp/__init__.py def base_interpreter(prog): """# doeff: interpreter, default""" - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) # doeff: default base_env = Program.pure({'debug': True}) @@ -538,7 +540,7 @@ myapp/ # myapp/__init__.py def base_interpreter(prog): """# doeff: interpreter, default""" - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) # doeff: default base_env = Program.pure({ @@ -714,7 +716,7 @@ graph TD # myapp/__init__.py def base_interpreter(prog): """# doeff: interpreter, default""" - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) ``` 2. **Or specify manually:** @@ -929,7 +931,7 @@ def my_interpreter(prog: Program): # doeff: interpreter, default """ - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) ``` **Recommended: Preceding comment for envs** @@ -1042,12 +1044,12 @@ grep -r "def.*interpreter" --include="*.py" ```python # Before def my_interpreter(prog: Program): - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) # After def my_interpreter(prog: Program): """# doeff: interpreter, default""" - return run(prog, handlers=default_handlers()).value + return run(scheduled(writer(state()(reader(env={})(prog))))) ``` **Step 3: Test discovery** diff --git a/docs/15-cli-script-execution.md b/docs/15-cli-script-execution.md index 3af96a3f4..f83221968 100644 --- a/docs/15-cli-script-execution.md +++ b/docs/15-cli-script-execution.md @@ -161,10 +161,12 @@ The following variables are automatically injected into your script's global nam | Variable | Type | Description | |----------|------|-------------| -| `RunResult` | `Type[RunResult]` | RunResult class for type checking | | `Program` | `Type[Program]` | Program class for type checking | -| `run` | `Callable[..., RunResult[T]]` | Runtime entrypoint for executing programs | -| `default_handlers` | `Callable[[], dict]` | Builds the default sync handler set | +| `run` | `Callable[..., T]` | Runtime entrypoint for executing programs (returns raw value) | +| `reader` | `Callable` | Reader handler from `doeff_core_effects.handlers` | +| `state` | `Callable` | State handler from `doeff_core_effects.handlers` | +| `writer` | `Callable` | Writer handler from `doeff_core_effects.handlers` | +| `scheduled` | `Callable` | Scheduler from `doeff_core_effects.scheduler` | ### Standard Library @@ -192,8 +194,8 @@ if callable(interpreter): print(f"New result: {new_result}") # Use runtime helpers -runtime_result = run(program, handlers=default_handlers()) -print(f"Runtime helper result: {runtime_result.value}") +runtime_result = run(scheduled(writer(state()(reader(env={})(program))))) +print(f"Runtime helper result: {runtime_result}") # Use standard library print(f"Python version: {sys.version}") @@ -247,8 +249,8 @@ Modify and re-execute programs: ```bash uv run doeff run --program myapp.base_program - <<'PY' # Re-run with runtime helpers -result = run(program, handlers=default_handlers()) -print(f"Re-run result: {result.value}") +result = run(scheduled(writer(state()(reader(env={})(program))))) +print(f"Re-run result: {result}") PY ``` diff --git a/docs/17-effect-boundaries.md b/docs/17-effect-boundaries.md index 28381d24e..45a7d870b 100644 --- a/docs/17-effect-boundaries.md +++ b/docs/17-effect-boundaries.md @@ -13,7 +13,6 @@ doeff for Python async integration. - [Scheduler Suspension vs VM Escape](#scheduler-suspension-vs-vm-escape) - [Formal Model](#formal-model) - [VM Parameterization](#vm-parameterization) -- [Runner Pairing (`run` vs `async_run`)](#runner-pairing-run-vs-async_run) - [Custom Handler Rules](#custom-handler-rules) ## Boundary Model @@ -31,7 +30,7 @@ responsibilities. Most effects are fully handled by doeff handlers and never leave the VM stepping loop: - `Ask`, `Local` -- `Get`, `Put`, `Modify` +- `Get`, `Put` - `Tell`, `Listen` - `Try` - scheduler effects such as `Spawn`, `Wait`, `Gather`, `Race` @@ -58,9 +57,8 @@ All of those effects are resolved by handlers inside doeff. | Category | Effects | Boundary | | --- | --- | --- | | Context | `Ask`, `Local` | Inside doeff | -| State / Writer / Result | `Get`, `Put`, `Modify`, `Tell`, `Listen`, `Try` | Inside doeff | +| State / Writer / Result | `Get`, `Put`, `Tell`, `Listen`, `Try` | Inside doeff | | Scheduler | `Spawn`, `Wait`, `Gather`, `Race` | Inside doeff | -| Cache | `CacheGet`, `CachePut`, `CacheDelete`, `CacheExists` | Inside doeff | | Promise | `CreatePromise`, `CompletePromise`, `FailPromise` | Inside doeff | | Async handoff | `Await` via `async_await_handler` | Escapes as `PythonAsyncSyntaxEscape` | @@ -83,7 +81,8 @@ Key points: - `await` cannot be evaluated inside a normal sync function without handing control to an event loop. -- `async_run(...)` is the opt-in API that allows this handoff. +- `run(scheduled(prog))` is the entry point that allows this handoff when async handlers are + composed into the program. - The escape type is not a general-purpose "leave doeff" mechanism; it is specifically for Python async integration. @@ -117,50 +116,31 @@ StepOutcome = Done | Failed | Continue ## VM Parameterization The VM remains a pure step machine and does not require `VM[M]` parameterization. -State, reader, writer, scheduler, cache, and promise operations are interpreted by handlers and +State, reader, writer, scheduler, and promise operations are interpreted by handlers and return normal VM outcomes (`Continue`, `Done`, `Failed`) inside doeff. -Only Python `await` syntax forces an external boundary, so doeff keeps one VM and exposes two -interpreters: `run(...)` for sync execution and `async_run(...)` for event-loop integration. +Only Python `await` syntax forces an external boundary, so doeff keeps one VM and one +entry point: `run(doexpr)` which accepts a single program argument and returns the raw value. -## Runner Pairing (`run` vs `async_run`) - -Use runner + handler presets intentionally: +## Custom Handler Rules -Examples in this chapter use `handlers=` only to install the builtin runtime preset. For custom -handler composition, call a Program -> Program handler installer directly, for example -`handler(program)`. +When adding handlers, keep the boundary strict: -| Entry point | Preset | Await behavior | -| --- | --- | --- | -| `run(...)` | `default_handlers()` | `sync_await_handler` bridges awaitables via background loop thread | -| `await async_run(...)` | `default_async_handlers()` | `async_await_handler` emits `PythonAsyncSyntaxEscape` for async driver path | +1. Handle domain effects inside handlers whenever possible. +2. Use `PythonAsyncSyntaxEscape` only for operations that must execute in Python async context. +3. Keep scheduler state transitions internal to scheduler handlers. Example: ```python -import asyncio -from doeff import Await, async_run, default_async_handlers, default_handlers, do, run +from doeff import do, run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled @do def compute(): - value = yield Await(asyncio.sleep(0.01, result=21)) + value = yield Ask("key") return value * 2 -sync_result = run(compute(), handlers=default_handlers()) -assert sync_result.value == 42 - -async def main(): - async_result = await async_run(compute(), handlers=default_async_handlers()) - assert async_result.value == 42 - -asyncio.run(main()) +result = run(scheduled(writer(state()(reader({"key": 21})(compute()))))) +assert result == 42 ``` - -## Custom Handler Rules - -When adding handlers, keep the boundary strict: - -1. Handle domain effects inside handlers whenever possible. -2. Use `PythonAsyncSyntaxEscape` only for operations that must execute in Python async context. -3. Keep scheduler state transitions internal to scheduler handlers. -4. Pair sync and async runners with their matching handler presets. diff --git a/docs/18-effect-combinations.md b/docs/18-effect-combinations.md index 319b4b073..e2cf3a42f 100644 --- a/docs/18-effect-combinations.md +++ b/docs/18-effect-combinations.md @@ -16,12 +16,12 @@ contracts. The matrix below follows `SPEC-EFF-100` outer/inner structure. -| Outer / Inner | Ask | Get | Put | Log | Try | Local | Listen | WithIntercept | Gather | -|---------------|-----|-----|-----|-----|------|-------|--------|-----------|--------| +| Outer / Inner | Ask | Get | Put | Tell | Try | Local | Listen | WithObserve | Gather | +|---------------|-----|-----|-----|------|------|-------|--------|-------------|--------| | **Local** | Scoped | Propagates | Propagates | Propagates | Propagates | Scoped | Propagates | Propagates | Propagates | | **Listen** | - | - | - | Captured* | - | - | Nested | - | All captured* | | **Try** | - | - | Persists | Persists | Wrapped | Restores | - | - | First error | -| **WithIntercept** | Transform | Transform | Transform | Transform | Transform | Transform | Transform | Transform | Transform | +| **WithObserve** | Transform | Transform | Transform | Transform | Transform | Transform | Transform | Transform | Transform | | **Gather** | Inherit | Shared** | Shared** | Merged | Isolated | Inherit | Shared | Propagates | Nested | `*` Listen captures logs only on success paths. On error, the error propagates and logs are not @@ -37,7 +37,7 @@ wrapped in `ListenResult`.[^D5] - `Persists`: Side effects persist even when errors are caught. - `Wrapped`: Result is wrapped in outer effect type. - `Restores`: Environment restores to pre-scope value. -- `Transform`: Effect can be transformed by `WithIntercept` (including nested programs).[^D1] +- `Transform`: Effect can be transformed by `WithObserve` (including nested programs).[^D1] - `Isolated`: Error handling remains per-branch while parent receives failing outcome. - `Inherit`: Child inherits parent environment snapshot at invocation time. - `Shared`: All branches use the same underlying resource. @@ -109,17 +109,19 @@ def test_law_5(): # Environment is restored after Try exits ``` -### Law 6: WithIntercept Transformation Law +### Law 6: WithObserve Transformation Law -`WithIntercept` transforms effects in the intercepted program, including nested program values in +`WithObserve` transforms effects in the observed program, including nested program values in payloads (for example `Gather` children and `Local`/`Listen` sub-programs).[^D1] ```python +from doeff import WithObserve + @do def test_law_6(): seen = [0] - def transform(effect): + def observer(effect): if isinstance(effect, AskEffect): seen[0] += 1 return effect @@ -128,7 +130,7 @@ def test_law_6(): def child(): return (yield Ask("key")) - _ = yield WithIntercept(transform, Gather(child(), child()), types=(AskEffect,), mode="include") + _ = yield WithObserve(observer, Gather(child(), child())) assert seen[0] == 2 ``` @@ -197,20 +199,22 @@ def test_law_9(): ## Sync vs Async Runtime Differences `SPEC-EFF-100` defines shared semantics plus explicit runtime-dependent behavior. +The single entry point is `run(doexpr)` — compose `scheduled()` into the program to enable +scheduler-based concurrency. -| Concern | `run(...)` (`SyncRuntime`) | `async_run(...)` (`AsyncRuntime`) | +| Concern | Sequential (no scheduler) | Concurrent (`scheduled()`) | | --- | --- | --- | -| Gather execution model | Sequential child execution. | Parallel child execution. | +| Gather execution model | Sequential child execution. | Concurrent child execution via scheduler. | | Gather store semantics | Shared store, deterministic state observation by child order. | Shared store, race-prone interleavings for read/modify/write patterns. | | Gather error propagation | Stop at first failing child; later children are not executed. | First failure propagates; other children may continue running. | | Gather side effects on sibling failure | Effects only from children that already ran before failure. | Effects from siblings may still happen after failure is observed. | -Guideline: treat `Gather` as shared-store in both runtimes; require explicit coordination for -race-sensitive async state updates. +Guideline: treat `Gather` as shared-store in both modes; require explicit coordination for +race-sensitive concurrent state updates. ## Design Decision Notes -[^D1]: **D1 WithIntercept Scope**: WithIntercept applies structural rewriting to nested programs in +[^D1]: **D1 WithObserve Scope**: WithObserve applies structural rewriting to nested programs in effect payloads. Scope across task boundaries is runtime-dependent; in current implementation, it does not reach independently spawned tasks that do not share the parent continuation stack. diff --git a/docs/20-why-effects-over-di.md b/docs/20-why-effects-over-di.md index 851ea5b33..7ab96a3ec 100644 --- a/docs/20-why-effects-over-di.md +++ b/docs/20-why-effects-over-di.md @@ -71,13 +71,15 @@ def my_workflow(): return analysis, image # Stack handlers — routing is automatic +from doeff_core_effects.handlers import reader result = run( - seedream_handler( - gemini_handler( - openai_handler(my_workflow()), + reader({"openai_api_key": "...", "gemini_api_key": "...", "seedream_api_key": "..."})( + seedream_handler( + gemini_handler( + openai_handler(my_workflow()), + ), ), ), - env={"openai_api_key": "...", "gemini_api_key": "...", "seedream_api_key": "..."} ) ``` @@ -133,7 +135,7 @@ def cost_cap_handler(effect: LLMChat | LLMEmbedding | LLMStructuredOutput | Imag current_cost = yield Get("total_cost") if current_cost > MAX_BUDGET: raise BudgetExceededError(current_cost) - yield Pass() + yield Pass(effect, k) # One retry handler works for everything @do @@ -154,7 +156,7 @@ def retry_handler(effect: Effect, k): Effects can be intercepted, transformed, or blocked without the original code knowing. ```python -from doeff import Get, Pass, Resume, Tell, WithIntercept, do +from doeff import Get, Pass, Resume, Tell, WithObserve, do # Budget cap across ALL providers — impossible to bypass @do @@ -163,7 +165,7 @@ def budget_handler(effect: LLMChat | LLMStructuredOutput | ImageGenerate, k): if total > settings.max_budget_usd: yield Tell(f"Budget exceeded: ${total:.2f} > ${settings.max_budget_usd}") return (yield Resume(k, BudgetExceededResult())) - yield Pass() + yield Pass(effect, k) # Dry-run mode — intercept ALL side effects @do @@ -171,14 +173,13 @@ def dry_run_handler(effect: LLMChat | ImageGenerate, k): yield Tell(f"[DRY RUN] Would execute: {type(effect).__name__}(model={effect.model})") return (yield Resume(k, mock_response(effect))) -# Rewriting is better modeled with WithIntercept than Delegate() -@do +# Rewriting is better modeled with WithObserve def rewrite_gpt4(effect): if isinstance(effect, LLMChat) and effect.model == "gpt-4": return LLMChat(**{**vars(effect), "model": "gpt-4o-mini"}) return effect -rewritten_program = WithIntercept(rewrite_gpt4, my_workflow(), types=(LLMChat,), mode="include") +rewritten_program = WithObserve(rewrite_gpt4, my_workflow()) ``` With DI, each of these requires a new wrapper class per service type. @@ -241,7 +242,6 @@ result = run( gemini_handler( openai_handler(my_workflow()), # inner tries first ), - env={...} ) ``` @@ -265,13 +265,13 @@ The program never mentions a provider. The handler stack decides where secrets c ```python # Production: Google Cloud Secret Manager -run(gsm_handler(deploy_workflow()), env={...}) +run(gsm_handler(deploy_workflow())) # Local dev: 1Password -run(onepassword_handler(deploy_workflow()), env={...}) +run(onepassword_handler(deploy_workflow())) # CI: AWS Secrets Manager -run(aws_secrets_handler(deploy_workflow()), env={...}) +run(aws_secrets_handler(deploy_workflow())) # Fallback chain: try 1Password, fall back to env vars run( @@ -289,13 +289,11 @@ With DI, you'd need a `SecretService` protocol, provider implementations, a fact # Production result = run( openai_production_handler(my_workflow()), - env={"openai_api_key": real_key} ) # Test — swap handler, everything else identical result = run( openai_mock_handler(my_workflow()), - store={"mock_config": MockOpenAIConfig(default_response="test")} ) # Replay — record and replay real responses @@ -353,7 +351,7 @@ With effects, dependencies are resolved at runtime by whatever handler is stacke ### 4. Static Analysis -Type checkers can fully verify DI wiring. Effects with `Delegate()` chains are harder to statically verify — you trust the handler stack at runtime. +Type checkers can fully verify DI wiring. Effects with `Pass()` chains are harder to statically verify — you trust the handler stack at runtime. ## Decision Guide @@ -447,7 +445,8 @@ This program uses three independent effect domains: from doeff_sim.handlers import deterministic_sim_handler from doeff_events.handlers import event_handler from doeff_time.handlers import async_time_handler -from doeff import async_run, default_async_handlers, run +from doeff import run +from doeff_core_effects.scheduler import scheduled # Deterministic simulation (instant, reproducible) result = run( @@ -458,12 +457,13 @@ result = run( ), ) -# Same program, real-time execution (wall-clock, asyncio) -result = await async_run( - async_time_handler()( # asyncio.sleep, time.time() - event_handler()(trading_strategy()), # Same event handler +# Same program, real-time execution (with scheduler for concurrency) +result = run( + scheduled( + async_time_handler()( # asyncio.sleep, time.time() + event_handler()(trading_strategy()), # Same event handler + ), ), - handlers=default_async_handlers(), ) ``` diff --git a/docs/22-with-intercept.md b/docs/22-with-intercept.md deleted file mode 100644 index 10e8c3ace..000000000 --- a/docs/22-with-intercept.md +++ /dev/null @@ -1,387 +0,0 @@ -# WithIntercept — Cross-Cutting Effect Observation - -`WithIntercept` lets you observe and transform **every** effect yielded inside a scope — including effects emitted by handlers themselves. This is the primitive you reach for when logging, tracing, or auditing needs to see the full picture, not just the user program's yields. - -## The Problem - -Handler-emitted effects are invisible to observers below the handler in the stack. - -Consider a writer handler that consumes `Tell`, and a handler_A that internally yields `Tell("from A")`. An observer sitting below handler_A never sees that internal `Tell` — it travels **up** to the writer, bypassing everything below: - -``` -┌──────────────────────┐ -│ writer (Tell) │ <- consumes Tell -├──────────────────────┤ -│ handler_A │ <- internally yields Tell("from A") -> goes UP -├──────────────────────┤ -│ printer (observer) │ <- never sees handler_A's Tell -├──────────────────────┤ -│ user_program │ <- its Tell is seen by printer -└──────────────────────┘ -``` - -The user program's `Tell` passes through the printer on its way up, so the printer sees it. But handler_A's `Tell` originates *above* the printer and goes further up — the printer is never in the path. - -The legacy `Intercept` API was removed because it could not implement cross-cutting concerns like "log every `Tell` in this subtree, regardless of who emits it." `WithIntercept` solves this by operating at the VM level, where it sees yields from handlers inside its scope. - -## API - -```python -from doeff import WithIntercept - -WithIntercept(f, expr, types=None, mode="include") -``` - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `f` | `@do (effect: Effect) -> DoExpr` | *(required)* | Interceptor function. Receives matched effects, must return a `DoExpr` (typically the original effect unchanged, or a transformed replacement). | -| `expr` | `DoExpr` | *(required)* | The scoped program to observe. All yields within this subtree are candidates for interception. | -| `types` | `tuple[type, ...]` or `None` | `None` | Optional effect or `DoCtrl` subclass filter. `None` means no type filter. | -| `mode` | `"include"` or `"exclude"` | `"include"` | How `types` is interpreted when a filter is provided. See **Type Filtering** below. | - -**Return**: A `DoExpr` that, when interpreted, runs `expr` while routing matched yields through `f`. - -## Basic Usage - -The simplest use: observe every effect in a scope without changing anything. - -```python -from dataclasses import dataclass -from doeff import Effect, EffectBase, WithIntercept, do, EffectGenerator, run, default_handlers, Tell, WriterTellEffect - -observed: list[str] = [] - -@do -def log_all(effect: Effect): - """Interceptor that records effects, then passes them through unchanged.""" - observed.append(repr(effect)) - return effect # return the original effect — no transformation - -@do -def my_program() -> EffectGenerator[None]: - yield Tell("hello") - yield Tell("world") - -result = run( - WithIntercept(log_all, my_program(), types=(WriterTellEffect,), mode="include"), - handlers=default_handlers(), -) - -# observed now contains repr strings for both Tell effects -``` - -The interceptor `log_all` sees each `Tell` as it passes through, records it, and returns the original effect so dispatch continues normally. - -## Cross-Cutting Observation - -This is the key property that distinguishes `WithIntercept` from legacy interception wrappers. - -When a handler inside the scope yields effects of its own, `WithIntercept` sees those too. This enables true cross-cutting observation — you get a complete trace of every effect in the subtree, not just the ones the user program directly yields. - -```python -from dataclasses import dataclass -from doeff import ( - Effect, - EffectBase, - Resume, - Tell, - WithIntercept, - WriterTellEffect, - default_handlers, - do, - run, - EffectGenerator, -) - -@dataclass(frozen=True) -class Ping(EffectBase): - label: str - -seen: list[str] = [] - -@do -def observe_tells(effect: Effect): - """Cross-cutting observer: sees Tell from ANY source.""" - if isinstance(effect, WriterTellEffect): - seen.append(effect.message) - return effect - -# Assume ping_handler is a Program -> Program handler installer, for example one -# produced by defhandler. It internally yields Tell while handling Ping. - -@do -def user_program() -> EffectGenerator[str]: - yield Tell("from-user") - result: str = yield Ping(label="foo") - yield Tell("after-ping") - return result - -# WithIntercept wraps the handler scope, so f sees handler's yields. -observed_program = WithIntercept( - observe_tells, - ping_handler(user_program()), - types=(WriterTellEffect,), - mode="include", -) - -run(observed_program, handlers=default_handlers()) - -# seen == ["from-user", "handler:foo", "after-ping"] -# ^^^^^^^^^^^ -# This one comes from ping_handler, not user_program. -# WithIntercept catches it because interception is VM-scoped. -``` - -## Type Filtering - -The `types` and `mode` parameters control which yields reach `f`. - -### Include Mode (default) - -Only effects matching one of the listed types fire `f`: - -```python -# Only intercept Tell effects -WithIntercept(f, expr, types=(WriterTellEffect,), mode="include") -``` - -With an empty `types` tuple in include mode, **nothing matches** — `f` never fires: - -```python -# f is never called (include + empty = match nothing) -WithIntercept(f, expr, types=(), mode="include") -``` - -### Exclude Mode - -All effects fire `f` **except** those matching the listed types: - -```python -# Observe everything EXCEPT Tell -WithIntercept(f, expr, types=(WriterTellEffect,), mode="exclude") -``` - -With an empty `types` tuple in exclude mode, **everything matches** — `f` fires on every yield: - -```python -# f is called on every yielded effect (exclude + empty = match everything) -WithIntercept(f, expr, types=(), mode="exclude") -``` - -### Summary Table - -| `mode` | `types` | What matches | -|--------|---------|-------------| -| `"include"` | `()` | Nothing | -| `"include"` | `(A, B)` | Only `A` and `B` instances | -| `"exclude"` | `()` | Everything | -| `"exclude"` | `(A, B)` | Everything except `A` and `B` | - -The `isinstance` check applies to the yielded value, so subclass relationships work as expected. Filtering on `WriterTellEffect` matches `Tell` because `Tell` is a subclass of `WriterTellEffect`. - -## Intercepting Control Flow - -Type filtering works on `DoCtrl` types too, not just `Effect` subclasses. This lets you observe structural control flow — handler installations, resumptions, and delegations. - -```python -from doeff import Effect, WithHandlerType, WithIntercept, Resume, Delegate, do - -ctrl_log: list[str] = [] - -@do -def log_ctrl(effect: Effect): - ctrl_log.append(type(effect).__name__) - return effect - -# Observe only handler-scope and Resume control nodes -observed = WithIntercept( - log_ctrl, - my_handler(my_program()), - types=(WithHandlerType, Resume), - mode="include", -) -``` - -This is useful for debugging handler dispatch — you can see exactly when handlers are installed and when continuations are resumed. - -## Effect Transformation - -`f` can return a **different** effect than the one it received. The returned effect replaces the original in the dispatch pipeline. - -```python -from doeff import Effect, Tell, WithIntercept, WriterTellEffect, do - -@do -def redact_tells(effect: Effect): - """Replace Tell payloads with a redacted version.""" - if isinstance(effect, WriterTellEffect): - return Tell("[REDACTED]") - return effect - -redacted = WithIntercept( - redact_tells, - user_program(), - types=(WriterTellEffect,), - mode="include", -) -``` - -After `redact_tells` runs, the handler stack sees `Tell("[REDACTED]")` instead of the original payload. The user program is unaware — it still yields normally. - -## Effectful Interceptors - -The interceptor `f` can itself yield effects. This makes it possible to perform logging, metrics collection, or other side effects during observation. - -```python -from doeff import Effect, Tell, do - -@do -def audit_interceptor(effect: Effect): - """Interceptor that emits its own Tell for each observed effect.""" - yield Tell(f"audit: saw {type(effect).__name__}") - return effect # pass through the original - -audited = WithIntercept( - audit_interceptor, - ping_handler(user_program()), - types=(Ping,), - mode="include", -) -``` - -When `audit_interceptor` sees a `Ping`, it yields a `Tell` (which goes through normal handler dispatch) and then returns the original `Ping` for continued processing. - -## No Re-Entrancy - -When `f` yields effects, those yields **skip the interceptor that invoked f**. This prevents infinite loops and matches the semantics of handler re-entrancy. - -```python -from doeff import Effect, Tell, WithIntercept, do - -seen_by_f: list[str] = [] - -@do -def counting_interceptor(effect: Effect): - """Yields a Tell, but that Tell does NOT re-enter this interceptor.""" - seen_by_f.append(repr(effect)) - yield Tell("from-interceptor") # <- this Tell is NOT seen by counting_interceptor - return effect - -observed = WithIntercept( - counting_interceptor, - user_program(), - types=(), - mode="exclude", # match everything -) -``` - -Even though `counting_interceptor` matches all types and yields a `Tell`, that `Tell` bypasses itself. Without this rule, the interceptor would see its own `Tell`, yield another `Tell`, see that one, and loop forever. - -## Nesting - -Multiple `WithIntercept` layers compose naturally. Each layer's yields skip only its own interceptor — other layers still see them. - -```python -from doeff import Effect, Tell, WithIntercept, do - -inner_seen: list[str] = [] -outer_seen: list[str] = [] - -@do -def inner_observer(effect: Effect): - inner_seen.append(f"inner:{effect}") - yield Tell("from-inner-observer") # outer sees this, inner does not - return effect - -@do -def outer_observer(effect: Effect): - outer_seen.append(f"outer:{effect}") - return effect - -nested = WithIntercept( - outer_observer, - WithIntercept( - inner_observer, - user_program(), - types=(), mode="exclude", - ), - types=(), mode="exclude", -) -``` - -Here: -- `inner_observer` sees effects from `user_program` -- `inner_observer`'s `Tell("from-inner-observer")` skips itself but is seen by `outer_observer` -- `outer_observer` sees effects from both `user_program` and `inner_observer` - -## Scoping Rules - -`WithIntercept` sees **all** effects in the causal chain — including effects emitted by handlers that are processing effects from within the interceptor's scope. The placement of `WithIntercept` relative to handler installer calls does **not** affect visibility. - -```python -# Both arrangements are equivalent — f sees ping_handler's Tell yields in either case. - -# Arrangement A: WithIntercept outside -WithIntercept( - f, - ping_handler(user_program()), - types=(WriterTellEffect,), - mode="include", -) - -# Arrangement B: WithIntercept inside -ping_handler( - WithIntercept(f, user_program(), types=(WriterTellEffect,), mode="include"), -) -``` - -When `user_program` yields `Ping("x")`, the effect passes through the interceptor (filtered out by `types`) and reaches `ping_handler`. When `ping_handler` internally yields `Tell("handler:x")`, the interceptor sees it because the handler is processing an effect that originated from within the interceptor's scope. - -The only effects `f` does **not** see are its own yields (re-entrancy guard) and effects from completely unrelated dispatch chains. - -## WithIntercept vs Legacy Intercept (Removed) - -| Aspect | Legacy `Intercept` (removed) | `WithIntercept` | -|--------|-------------|-----------------| -| Level | Python InterceptFrame | VM-level `DoCtrl` node | -| Sees handler yields | No | Yes | -| Type filtering | No (manual `isinstance` in transform) | Built-in `types` + `mode` | -| Transform signature | `(effect) -> Effect \| Program \| None` | `(effect) -> DoExpr` | -| Can yield effects | No | Yes (`f` is a generator) | -| Re-entrancy guard | N/A | `f`'s yields skip its own interceptor | -| Filters on DoCtrl types | No | Yes (`WithHandlerType`, `Resume`, `Delegate`) | -| Propagates to Gather/Spawn | Yes | Scoped to `expr` subtree | -| Composability | First non-None transform wins | All layers compose independently | - -The legacy `Intercept` API is removed. Use `WithIntercept` for scoped interception. - -## Best Practices - -### DO: - -- **Use `WithIntercept` for observability** — tracing, logging, metrics, auditing. It was designed for cross-cutting concerns. -- **Filter with `types`** — narrow the interceptor to only the effects it cares about. An unfiltered interceptor on a hot path adds overhead. -- **Return the original effect** when you only want to observe. Unnecessary transformations make debugging harder. -- **Place `WithIntercept` anywhere in the handler stack** — it sees handler-emitted effects regardless of nesting position. -- **Keep interceptors simple** — an interceptor that yields many effects of its own becomes hard to reason about. - -### DON'T: - -- **Don't use `WithIntercept` as a handler replacement** — it does not consume effects. Effects always proceed through normal dispatch after `f` returns. -- **Don't rely on execution order between nested interceptors** for correctness — use them for observation, not orchestration. -- **Don't mutate shared state in `f` without synchronization** if the observed program uses `gather` for concurrency. -- **Don't use `exclude` mode with empty `types` in production** unless you genuinely need to intercept every single yield. It is useful for debugging but noisy at scale. -- **Don't assume nesting order matters for visibility** — `WithIntercept` sees handler-emitted effects regardless of position. Nesting order only affects which interceptor layer sees effects first when multiple `WithIntercept` layers are composed. - -## Summary - -| What | How | -|------|-----| -| Observe all effects in a scope | `WithIntercept(f, expr, types=(), mode="exclude")` | -| Observe specific effect types | `WithIntercept(f, expr, types=(Tell, Ping), mode="include")` | -| Observe everything except certain types | `WithIntercept(f, expr, types=(Tell,), mode="exclude")` | -| See handler-emitted effects | Automatic — `WithIntercept` sees them regardless of nesting | -| Transform effects before dispatch | Return a different effect from `f` | -| Emit side effects during observation | `yield` inside `f` | -| Avoid infinite loops | Automatic — `f`'s yields skip its own interceptor | -| Compose multiple observers | Nest `WithIntercept` layers; each is independent | diff --git a/docs/MARKERS.md b/docs/MARKERS.md index 16beb0f0d..b866bb738 100644 --- a/docs/MARKERS.md +++ b/docs/MARKERS.md @@ -9,8 +9,10 @@ Doeff markers are special comments that allow precise categorization of function Markers use the format `# doeff: ` and can be placed on or near function definitions: ```python -def my_interpreter(program: Program): # doeff: interpreter - return program.run() +from doeff import run + +def my_interpreter(program): # doeff: interpreter + return run(program) ``` ## Supported Markers @@ -19,15 +21,16 @@ def my_interpreter(program: Program): # doeff: interpreter 1. **`interpreter`** - Functions that execute/interpret Program objects ```python - def run_program(program: Program): # doeff: interpreter - return program.run() + from doeff import run + + def run_program(program): # doeff: interpreter + return run(program) ``` 2. **`transform`** / **`transformer`** - Functions that transform Program objects ```python - @do - def optimize(program: Program) -> Program: # doeff: transform - return program.optimize() + def add_logging(program): # doeff: transform + return writer(program) ``` 3. **`kleisli`** - Composable effect-handling functions (usually with `@do` decorator) @@ -43,15 +46,19 @@ def my_interpreter(program: Program): # doeff: interpreter For interpreters (requires both `interpreter` and `default`): ```python - def my_interpreter(program: Program): + from doeff import run + + def my_interpreter(program): """# doeff: interpreter, default""" - return program.run() + return run(program) ``` For environments: ```python + from doeff import Pure + # doeff: default - base_env: Program[dict] = Program.pure({ + base_env = Pure({ 'db_host': 'localhost', 'timeout': 10 }) @@ -65,8 +72,8 @@ Functions can have multiple roles specified with comma-separated markers: ```python @do -def hybrid_function(program: Program): # doeff: kleisli, transform - transformed = yield program.transform() +def hybrid_function(program): # doeff: kleisli, transform + transformed = yield program return transformed ``` @@ -76,14 +83,14 @@ Markers can be placed in several positions: ### Same Line as Function Definition ```python -def interpreter(program: Program): # doeff: interpreter +def interpreter(program): # doeff: interpreter pass ``` ### On Multi-line Function Signatures ```python def interpreter( # doeff: interpreter - program: Program, + program, config: dict = None ): pass @@ -92,7 +99,7 @@ def interpreter( # doeff: interpreter ### Inline with Parameters ```python def interpreter( - program: Program, # doeff: interpreter + program, # doeff: interpreter verbose: bool = False ): pass @@ -144,33 +151,38 @@ Source Code → Indexer Extracts Markers → Index Entry with Markers → IDE Pl ### 1. Be Explicit Mark functions when their role is clear: ```python +from doeff import run + # Good - explicit marking -def my_interpreter(program: Program): # doeff: interpreter - return program.run() +def my_interpreter(program): # doeff: interpreter + return run(program) # Less ideal - relies on type detection -def my_interpreter(program: Program): - return program.run() +def my_interpreter(program): + return run(program) ``` ### 2. Use Consistent Placement Choose a consistent marker placement style in your codebase: ```python # Style 1: Same line -def func(program: Program): # doeff: interpreter +def func(program): # doeff: interpreter # Style 2: Multi-line signature def func( # doeff: interpreter - program: Program + program ): ``` ### 3. Mark Factory Functions Mark functions that create interpreters/transforms: ```python +from doeff import run +from doeff_core_effects.handlers import reader + def create_interpreter(config: dict): # doeff: interpreter - def inner(program: Program): - return program.run_with_config(config) + def inner(program): + return run(reader(config)(program)) return inner ``` @@ -179,7 +191,7 @@ Add docstrings explaining complex marker usage: ```python @do def complex_function( # doeff: kleisli, transform - program: Program + program ): """ This function acts as both a Kleisli arrow and a transformer. @@ -192,34 +204,41 @@ def complex_function( # doeff: kleisli, transform ### Basic Interpreter ```python -def simple_interpreter(program: Program): # doeff: interpreter +from doeff import run + +def simple_interpreter(program): # doeff: interpreter """Execute a program with default settings.""" - return program.run() + return run(program) ``` -### Async Interpreter +### Interpreter with Handlers ```python -from doeff import async_run, default_async_handlers +from doeff import run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled -async def async_interpreter( # doeff: interpreter - program: Program, - timeout: float = None +def full_interpreter( # doeff: interpreter + program, + env: dict = None ): - """Execute program asynchronously with optional timeout.""" - return await async_run(program, handlers=default_async_handlers()) + """Execute program with reader, state, writer, and scheduler.""" + wrapped = scheduled(writer(state()(reader(env or {})(program)))) + return run(wrapped) ``` ### Transform Chain ```python -@do +from doeff_core_effects.handlers import state, writer + def optimization_pipeline( # doeff: transform - program: Program, - level: int = 1 -) -> Program: - """Apply multiple optimization passes.""" - for _ in range(level): - program = program.optimize() - return program + program, + with_state: bool = True +): + """Apply handler wrapping as a transform.""" + wrapped = writer(program) + if with_state: + wrapped = state()(wrapped) + return wrapped ``` ### Kleisli Composition @@ -235,10 +254,13 @@ def data_pipeline(): # doeff: kleisli ### Class Method Interpreter ```python +from doeff import run +from doeff_core_effects.handlers import reader + class Executor: - def execute(self, program: Program): # doeff: interpreter + def execute(self, program): # doeff: interpreter """Execute program with executor context.""" - return program.run_with_context(self.context) + return run(reader(self.context)(program)) ``` ## Troubleshooting diff --git a/docs/MILESTONES.md b/docs/MILESTONES.md index dd9a01792..42c9cffcc 100644 --- a/docs/MILESTONES.md +++ b/docs/MILESTONES.md @@ -41,7 +41,7 @@ Must complete before anything goes public. Gates all other milestones. - [x] `README.md:340` — remove old project-name attribution line - [x] `docs/index.md:221` — same -- [x] `doeff/_vendor.py:2,4` — source attribution comments +- [x] `doeff/_vendor.py` — file has been removed (vendored code now lives in `doeff_vm`) - [x] `tests/misc/test_segmentation_pragmo.py.skip` — dead test with legacy imports → delete file ### M0.5 — Purge secrets @@ -64,7 +64,7 @@ Only 2 files outside VAULT: ### M0.8 — Drop phantom dependencies Zero-risk removals — these are never imported: -- [x] Remove `returns>=0.22.0` from `pyproject.toml` (0 imports — doeff has own `Result`/`Maybe` in `_vendor.py`) +- [x] Remove `returns>=0.22.0` from `pyproject.toml` (0 imports — doeff has own `Result`/`Maybe` via `doeff_vm`) - [x] Remove `cytoolz>=1.0.0` from `pyproject.toml` (0 imports) - [x] Move `beartype>=0.19.0` from `dependencies` to `[dependency-groups] dev` only (only used in 1 test file) - [x] Consider making `phart>=1.1.4` optional (already has `try/except ImportError` guard) @@ -226,11 +226,11 @@ No docs build system exists (no mkdocs.yml, no sphinx). | `09-advanced-effects.md` | LOW | 1 | | `13-api-reference.md` | LOW | 1 (runtime kwargs section) | -- [ ] Rewrite all examples to use `run()` / `async_run()` + `default_handlers()` / `default_async_handlers()` +- [ ] Rewrite all examples to use `run(doexpr)` with composed handler stacks (no `default_handlers()` — it was removed; `async_run()` was removed — use `run(scheduled(...))` instead) ### M4.3 — Fix naming inconsistency -- [ ] Standardize on `async_run` across all docs -- [ ] `01-getting-started.md` and `02-core-concepts.md` reference `async_run` +- [ ] Remove all `async_run` references — `async_run` was removed; use `run(scheduled(...))` instead +- [ ] `01-getting-started.md` and `02-core-concepts.md` may still reference `async_run` — update to `run(scheduled(...))` ### M4.4 — Write missing critical docs Biggest structural gaps: diff --git a/docs/cache.md b/docs/cache.md deleted file mode 100644 index 7f9b2bcf0..000000000 --- a/docs/cache.md +++ /dev/null @@ -1,72 +0,0 @@ -# Cache Effect - -This guide explains how to use the cache effect helpers (`CacheGet`, `CachePut`, and the -`@cache` decorator), which persist values with the built-in sqlite handler. - -When you decorate a function with `@cache`, the default key is a tuple of the function's fully -qualified module path, positional arguments, and keyword arguments wrapped in `FrozenDict`. Using the -module path (for example `"my_app.services.fetch_user"`) ensures cache hits remain valid across -separate Python processes. - -## Cache Policy Fields - -`CachePut` accepts either individual policy parameters or a pre-built `CachePolicy`. -All keyword parameters are optional: - -- `ttl`: Expiry in seconds. `None` means the entry never expires. Values \<= 0 are treated as - "no expiry" by the handler. -- `lifecycle`: Hint about how long data should live. Accepts the `CacheLifecycle` enum or the - strings `"transient"`, `"session"`, and `"persistent"`. Defaults to `TRANSIENT`. -- `storage`: Preferred storage backend hint. Accepts `CacheStorage` enum values or the strings - `"memory"` and `"disk"`. Defaults to `None`, in which case the lifecycle hint is used to derive - a suggestion via `CachePolicy.resolved_storage()`. -- `metadata`: Arbitrary mapping that is carried alongside the policy for custom handlers. -- `policy`: Either a `CachePolicy` instance or a mapping. When provided, the individual policy - parameters above must be omitted. - -## Interpreter Support Today - -The default `CacheEffectHandler` persists every entry in a sqlite database compressed with LZMA. -The database location can be overridden with the `DOEFF_CACHE_PATH` environment variable; when -absent it falls back to `${TMPDIR}/doeff_cache.sqlite3`. - -Currently the handler observes the policy fields as follows: - -- `ttl` is enforced. Entries expire once the handler observes a `ttl` > 0 and the deadline passes; - expired rows are deleted eagerly when accessed. -- `lifecycle` and `storage` are stored on the `CachePolicy` but only serve as hints today. - Regardless of their values, the bundled handler always uses the sqlite backend (disk storage). - Custom handlers may inspect `effect.policy` if you need different behaviour. -- `metadata` is preserved on the policy object and made available to custom handlers, but the - default handler ignores it. - -Because the built-in handler always uses sqlite, choosing `CacheLifecycle.PERSISTENT` or -`CacheStorage.DISK` does not change runtime behaviour at the moment; they are intended for -future extensions and for custom handler implementations. - -## Usage Example - -```python -from doeff import CachePut, CacheGet, CacheLifecycle, CacheStorage, default_handlers, run, do - -@do -def store_value(): - yield CachePut( - key=("query", {"user": 1}), - value={"result": "ok"}, - ttl=300.0, - lifecycle=CacheLifecycle.SESSION, - storage=CacheStorage.MEMORY, - metadata={"source": "demo"}, - ) - -@do -def load_value(): - return (yield CacheGet(("query", {"user": 1}))) - -run(store_value(), handlers=default_handlers()) -run(load_value(), handlers=default_handlers()) -``` - -The example sets every optional field to demonstrate the accepted values, even though only `ttl` -affects the bundled handler set today. diff --git a/docs/cli-run-command-architecture.md b/docs/cli-run-command-architecture.md index 7d708a4cb..78850dbe2 100644 --- a/docs/cli-run-command-architecture.md +++ b/docs/cli-run-command-architecture.md @@ -1,42 +1,46 @@ # doeff run Command Architecture -This note captures the refactored structure of the `doeff run` command introduced with the class-based CLI pipeline. Use it as a map when navigating or extending the workflow. +This note captures the structure of the `doeff run` command pipeline. Use it as a map when navigating or extending the workflow. ## Execution Flow - **RunContext** - - Collects the raw CLI arguments into a single data object. -- **RunCommand** - - Orchestrates the entire run lifecycle. - - Lazily initializes shared services (discovery, mergers) and the helper classes below. -- **SymbolResolver** - - Centralises cached imports and wraps validation helpers for programs, Kleisli transforms, and program transformers. -- **ProgramBuilder** - - Loads the target program. - - Merges environments and injects them with `Local`. - - Applies optional Kleisli and transformer stages sequentially. -- **RunServices** - - Hosts CLI discovery dependencies (`IndexerBasedDiscovery`, `StandardEnvMerger`, `StandardSymbolLoader`). -- **Interpreter Execution** - - Runs the interpreter, coercing callables into `RunResult` values when necessary. -- **_render_run_output** - - Emits user-facing output in text or JSON while handling optional reports and call-tree visuals. + - Collects the raw CLI arguments into a single data object (`program_path`, `interpreter_path`, `env_paths`, `set_vars`, `apply_paths`, `transformer_paths`, `output_format`). +- **resolve_context(ctx: RunContext) -> ResolvedRunContext** + - Loads the target program via `import_symbol()`. + - Auto-discovers the interpreter (via `IndexerBasedDiscovery`) if none was specified. + - Auto-discovers default envs if none were specified. + - Falls back to `default_interpreter` when no interpreter is found. +- **execute(resolved: ResolvedRunContext) -> Any** + - Applies `--apply` (Kleisli) and `--transform` stages sequentially. + - Merges environment sources (from `~/.doeff.py`, discovered, and explicit `--env` paths) via `StandardEnvMerger`. + - Applies `--set KEY=VALUE` overrides on top. + - Builds a `DoeffRunContext` for interpreters that support remote execution. + - Runs the interpreter, passing `env` and `ctx` kwargs if the interpreter's signature accepts them. +- **default_interpreter(program)** + - Composes the standard handler stack (`lazy_ask`, `state`, `writer`, `try_handler`, `slog_handler`, `listen_handler`, `await_handler`). + - Wraps with `scheduled()` and calls `run()`. +- **Output Rendering** + - Emits user-facing output in text or JSON format. + +## Supporting Classes + +- **DoeffRunContext** — Frozen dataclass capturing the original CLI invocation (program ref, interpreter ref, env refs, set overrides, apply/transform refs) for interpreters that need to reconstruct the `doeff run` command remotely (e.g. k3s Jobs, Docker). +- **RunnerContext** — Captures every source form (Python symbol path, inline `-c` Python, inline `--hy` Hy) plus `raw_argv` for remote backends. ## Mermaid Overview ```mermaid flowchart TD A[CLI Args] --> B[RunContext] - B --> C[RunCommand] - C --> D[SymbolResolver] - C --> E[RunServices] - E --> F[Discovery] - C --> G[ProgramBuilder] - F --> H[ResolvedRunContext] - G --> I[Program w/ Env & Effects] - C --> J[Interpreter Execution] - J --> K[RunExecutionResult] - K --> L[_render_run_output] + B --> C[resolve_context] + C --> D[ResolvedRunContext] + D --> E[execute] + E --> F[Apply --apply / --transform stages] + F --> G[Merge envs] + G --> H[Run interpreter] + H --> I[Return value] + I --> J[Output Rendering] ``` ## Communication Diagram @@ -44,40 +48,45 @@ flowchart TD ```mermaid sequenceDiagram participant CLI as CLI Entry - participant RC as RunCommand - participant SR as SymbolResolver - participant RS as RunServices - participant PB as ProgramBuilder + participant RS as resolve_context() + participant EX as execute() + participant DI as IndexerBasedDiscovery + participant IS as import_symbol() participant INT as Interpreter participant OUT as Output Renderer - CLI->>RC: Build RunContext - RC->>RS: Initialize discovery services (lazy) - RC->>RS: find_default_interpreter() - RS-->>RC: interpreter path - RC->>RS: discover_default_envs() - RS-->>RC: env path list - RC->>PB: load(program_path) - PB->>SR: resolve(program symbol) - SR-->>PB: Program - RC->>PB: inject_envs(env sources) - PB->>RS: merge_envs(sources) - PB-->>RC: Program with env - RC->>PB: apply kleisli/transformers - PB->>SR: resolve(kleisli/transformer) - SR-->>PB: callable - PB-->>RC: Final Program - RC->>SR: resolve(interpreter symbol) - SR-->>RC: Interpreter callable/instance - RC->>INT: run(program) - INT-->>RC: RunResult / value - RC->>OUT: Render with context & execution result + CLI->>RS: Build RunContext + RS->>IS: import_symbol(program_path) + IS-->>RS: Program + RS->>DI: find_default_interpreter(program_path) + DI-->>RS: interpreter path + RS->>DI: discover_default_envs(program_path) + DI-->>RS: env path list + RS-->>CLI: ResolvedRunContext + CLI->>EX: execute(resolved) + EX->>IS: import_symbol(apply/transform paths) + IS-->>EX: callables + EX->>EX: Merge envs (StandardEnvMerger) + EX->>IS: import_symbol(interpreter_path) + IS-->>EX: interpreter + EX->>INT: interpreter(program, env=..., ctx=...) + INT-->>EX: raw value + EX-->>CLI: result + CLI->>OUT: Render with context & result OUT-->>CLI: Text/JSON output ``` +## Key Implementation Details + +- The implementation lives in `doeff/cli/run_services.py`. +- `import_symbol()` supports both colon-separated (`module:attr`) and dotted (`module.attr`) paths. +- Hy import hook is activated at module load so `.hy` files can be resolved. +- `default_interpreter` composes handlers individually (there is no `default_handlers()` function). +- `run(doexpr)` takes a single argument and returns the raw value. + ## Extension Guidance -- Add new pre-run manipulations inside `ProgramBuilder` to keep run composition coherent. -- Extend auto-discovery logic via `RunServices` so dependent subsystems remain injectable. -- New output formats should be funneled through `_render_run_output` to share reporting logic. -- Maintain reusability: `SymbolResolver` should remain the single importer to avoid redundant module loads. \ No newline at end of file +- Add new pre-run manipulations as additional `--apply` or `--transform` stages. +- Extend auto-discovery logic via `IndexerBasedDiscovery` so dependent subsystems remain injectable. +- New output formats should be funneled through the output rendering layer. +- `import_symbol()` is the single importer to avoid redundant module loads. diff --git a/docs/documentation-fix-table.md b/docs/documentation-fix-table.md index cc5666a8d..0ccbbb2d3 100644 --- a/docs/documentation-fix-table.md +++ b/docs/documentation-fix-table.md @@ -1,14 +1,90 @@ # Documentation Update Checklist -This branch has applied the high-priority documentation fixes that were identified during the audit. +Last audit: 2026-07-08 -| Area | Status | Notes | +## Completed (this audit) + +| Area | File(s) | Notes | | --- | --- | --- | -| Public API cleanup | Done | Removed current-doc claims for `Safe`, `RuntimeResult`, `run_program()`, and `ProgramRunResult`. | -| Removed `IO(...)` effect | Done | Replaced the old `IO` chapter with a migration note and removed `IO` from reader-facing learning paths and examples. | -| Handler composition shape | Done | Reader-facing docs now use direct `handler(program)` calls and reserve the compatibility shim for deprecated usage. | -| `Pass()` vs `Delegate()` | Done | Transparent fallthrough examples now use `Pass()`. `Delegate()` is only described as the outer-result path. | -| Handler type filtering | Done | Added guidance that the raw handler's effect annotation becomes a runtime type filter inside the installed handler scope. | -| Runner guidance | Done | Reframed `handlers=` on `run()` / `async_run()` as a low-level runner hook rather than the primary custom-composition API. | -| Async cancellation wording | Done | Updated async docs to document `task.cancel()` instead of a fake top-level `Cancel(task)` constructor. | -| High-traffic examples | Done | Replaced deprecated handler-shim examples, stale handler guidance, and outdated snippets in README and the main guides. | +| OBSOLETE docs removed | `docs/cache.md`, `docs/08-graph-tracking.md`, `docs/22-with-intercept.md` | Entire files deleted — all referenced APIs (`cache`, `Step`, `Annotate`, `Snapshot`, `WithIntercept`) are `_Removed` | +| AGENTS.md project structure | `AGENTS.md` | Fixed references to non-existent `core.py`, `interpreter.py`, `effects/`, `handlers/`, `utils.py`, `types.py`, `cache.py`. Now describes actual layout (`program.py`, `do.py`, `run.py`, `result.py`, `handler_utils.py`, `cli/`) | +| AGENTS.md test example | `AGENTS.md` | `tests/test_cache.py::test_cache_eviction` replaced with existing `tests/test_core_effects.py::test_reader_ask` | +| README build command | `README.md` | `uv sync --reinstall` replaced with `make sync` + warning about stale Rust VM | +| README CLI flags | `README.md` | Deprecated flags (`--interpreter`, `--env`, `--apply`, `--transform`) marked as deprecated, `--hy` promoted | +| SPEC-WITH-INTERCEPT | `docs/specs/SPEC-WITH-INTERCEPT.md` | Added `WithObserve` pointer in existing DEPRECATED notice | +| index.md | `docs/index.md` | Removed links to deleted docs, fixed Quick Example (no `default_handlers`, correct `run()` usage), updated Effect Quick Reference, fixed code examples | +| API Reference | `docs/13-api-reference.md` | Full rewrite to match current API | + +## Fixed (Phase 3 — this audit) + +All previously listed STALE files have been fixed: + +### High Priority (tutorial/getting-started docs) — DONE + +| File | Fixes Applied | +| --- | --- | +| `docs/01-getting-started.md` | Removed `default_handlers`, `RunResult`, `async_run`, `KleisliProgram`, `Program.pure()` | +| `docs/02-core-concepts.md` | Removed `KleisliProgram`, `Delegate`, `async_run`, `WithIntercept`; fixed `Call` → `Expand` | +| `docs/03-basic-effects.md` | Removed `default_handlers`, `result.value`, `Modify` → `Get+Put`, `StructuredLog` → `slog` | +| `docs/04-async-effects.md` | Removed `async_run`, `default_async_handlers`; `task.cancel()` → `Cancel(task)` | +| `docs/05-error-handling.md` | Removed `RunResult` section; `isinstance(result, Ok)` for checking | + +### Medium Priority — DONE + +| File | Fixes Applied | +| --- | --- | +| `docs/07-cache-system.md` | Removed `@cache`; `TEMPORARY` → `TRANSIENT`; removed `DISTRIBUTED` | +| `docs/09-advanced-effects.md` | `Modify` → `Get+Put`; `WithIntercept` → `WithObserve`; `task.cancel()` → `Cancel(task)` | +| `docs/11-kleisli-arrows.md` | `KleisliProgram` → callable; `Call` → `Expand`; removed `.map/.flat_map/>>` | +| `docs/12-patterns.md` | Removed `AtomicGet/AtomicUpdate`, `Step`; fixed handler composition | +| `docs/17-effect-boundaries.md` | Removed `async_run`, `default_handlers`; fixed comparison table | +| `docs/18-effect-combinations.md` | `Log` → `Tell`; `WithIntercept` → `WithObserve` | +| `docs/20-why-effects-over-di.md` | `WithIntercept` → `WithObserve`; `Pass()` → `Pass(effect, k)` | +| `docs/MARKERS.md` | `program.run()` → `run(program)`; removed `async_run` | + +### Low Priority (design docs / proposals) — DONE + +| File | Fixes Applied | +| --- | --- | +| `docs/program-architecture-overview.md` | Removed `run(..., trace=True)`, `async_run`; updated to `WithObserve` | +| `docs/cli-run-command-architecture.md` | Updated to `RunContext`/`ResolvedRunContext`/`resolve_context()` | +| `docs/llm_unified_effects.md` | `LLMStructuredOutput` → `LLMStructuredQuery` | +| `docs/unified_image_effects.md` | `ImageResult` reclassified as result type | +| `docs/MILESTONES.md` | Updated `_vendor.py` references; noted API removals | +| `docs/specs/SPEC-WITHHANDLER-TYPE-FILTER.md` | `doeff/rust_vm.py` → `doeff/program.py`; fixed `EffectBase` import | +| `docs/proposals/001-doeff-flow-webui.md` | Added deprecation header; fixed code examples | +| `docs/proposals/002-run-result-printing-ownership-plan.md` | Marked superseded | +| `docs/12-mcp-tools.md` | `Sleep` → `ClockSleep` | + +### Additional fixes (discovered during verification) + +| File | Fixes Applied | +| --- | --- | +| `docs/06-io-effects.md` | Removed `default_handlers` reference | +| `docs/14-cli-auto-discovery.md` | 13 instances of `run(prog, handlers=default_handlers()).value` fixed | +| `docs/15-cli-script-execution.md` | Removed `RunResult`/`default_handlers` from variable table | +| `docs/gemini_cost_hook.md` | Removed `*default_handlers()` | +| `docs/gemini_client_setup.md` | `async_run` → `run(scheduled(...))` | +| `docs/seedream.md` | `async_run` → `run(scheduled(...))`; `Log` → `Tell` | +| `packages/doeff-llm/README.md` | `LLMStructuredOutput` → `LLMStructuredQuery` | + +## Cross-Cutting Deleted API Reference + +For anyone fixing the remaining docs, here are the key replacements: + +| Deleted API | Replacement | +| --- | --- | +| `default_handlers()` | Compose handlers individually: `writer(state()(prog))` | +| `run(prog, handlers=..., env=..., trace=...)` | `run(doexpr)` — single argument, returns raw value | +| `RunResult[T]` / `.value` / `.is_ok()` | `run()` returns the raw value directly | +| `async_run()` | `run(scheduled(prog))` | +| `Modify(key, f)` | `Get(key)` then `Put(key, f(val))` | +| `WithIntercept(f, expr, types=, mode=)` | `WithObserve(observer, body)` | +| `KleisliProgram` | `@do` returns a normal callable | +| `task.cancel()` | `yield Cancel(task)` | +| `Gather(*programs)` | `Spawn` each first, then `Gather(*tasks)` | +| `CacheGet` | `MemoGet` (in `doeff_core_effects.cache_effects`) | +| `Delegate` | `yield effect` to re-perform | +| `Program.pure(x)` | `Pure(x)` | +| `StructuredLog` | `slog(msg, **kwargs)` or `WriterTellEffect` | +| `graph_snapshot` / `graph_to_html` | Removed entirely | diff --git a/docs/gemini_client_setup.md b/docs/gemini_client_setup.md index c47e84e05..26b598cef 100644 --- a/docs/gemini_client_setup.md +++ b/docs/gemini_client_setup.md @@ -12,13 +12,15 @@ Provide `gemini_api_key`. Optional extras: Minimal example: ```python -from doeff import async_run, default_async_handlers - -result = await async_run( - my_program(), - handlers=default_async_handlers(), - env={"gemini_api_key": "your-api-key"}, -) +from doeff import do, run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled + +prog = my_program() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={"gemini_api_key": "your-api-key"})(prog) +result = run(scheduled(prog)) ``` ## Application Default Credentials (ADC) @@ -32,14 +34,14 @@ If `gemini_api_key` is absent, the client attempts ADC: Example with explicit project: ```python -result = await async_run( - my_program(), - handlers=default_async_handlers(), - env={ - "gemini_project": "your-project-id", - "gemini_location": "global", # optional override - }, -) +prog = my_program() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={ + "gemini_project": "your-project-id", + "gemini_location": "global", # optional override +})(prog) +result = run(scheduled(prog)) ``` ## Environment keys consulted diff --git a/docs/gemini_cost_hook.md b/docs/gemini_cost_hook.md index 23f76a811..04a5066c1 100644 --- a/docs/gemini_cost_hook.md +++ b/docs/gemini_cost_hook.md @@ -21,14 +21,18 @@ pricing for known models. ## Composition -Recommended stack: +Compose handlers around the program directly: ```python -handlers = [ - default_gemini_cost_handler, # built-in known-model pricing - custom_cost_handler, # optional overrides / unknown models - *default_handlers(), -] +from doeff import do, run +from doeff_core_effects.handlers import writer, state + +prog = default_gemini_cost_handler( # built-in known-model pricing + custom_cost_handler( # optional overrides / unknown models + writer(state()(program)) + ) +) +result = run(prog) ``` To fully replace default pricing, omit `default_gemini_cost_handler`. diff --git a/docs/index.md b/docs/index.md index c23062bf1..018e86643 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,10 +21,8 @@ Welcome to the comprehensive documentation for doeff - an algebraic effects syst 4. **[Async Effects](04-async-effects.md)** - Gather, Spawn, Await for async operations 5. **[Error Handling](05-error-handling.md)** - Result, Try for error handling 6. **[Cache System](07-cache-system.md)** - Cache effects with policies and handlers -7. **[Graph Tracking](08-graph-tracking.md)** - Execution tracking and visualization -8. **[Advanced Effects](09-advanced-effects.md)** - Gather, Atomic operations -9. **[Semaphore Effects](21-semaphore-effects.md)** - Create, acquire, and release permits with FIFO fairness -10. **[WithIntercept](22-with-intercept.md)** - Cross-cutting effect observation across handler boundaries +7. **[Advanced Effects](09-advanced-effects.md)** - Gather, concurrency patterns +8. **[Semaphore Effects](21-semaphore-effects.md)** - Create, acquire, and release permits with FIFO fairness ### Integration & Advanced Topics @@ -43,7 +41,6 @@ Welcome to the comprehensive documentation for doeff - an algebraic effects syst ### Specialized Topics - **[MARKERS.md](MARKERS.md)** - Marker-based Program manipulation -- **[cache.md](cache.md)** - Detailed cache system documentation - **[seedream.md](seedream.md)** - SeeDream integration - **[IDE Plugins](ide-plugins.md)** - PyCharm and VS Code extensions - **[Program Architecture](program-architecture-overview.md)** - Runtime internals overview @@ -72,7 +69,7 @@ Key characteristics: - **Algebraic effects with handlers**: Define effects as data, handle them with composable, swappable handlers - **One-shot continuations**: Each continuation resumes exactly once (unlike multi-shot systems like Koka or Eff) - **Rust VM runtime**: High-performance effect handling and continuation management -- **Batteries-included handlers**: Reader, State, Writer, Future, Result, Cache, Graph tracking — ready to use +- **Batteries-included handlers**: Reader, State, Writer, Scheduler, Result — ready to use - **Generator-based do-notation**: Write effectful code that looks like regular Python - **Stack-safe execution**: Trampolining prevents stack overflow - **Type safety**: Full type annotations with `.pyi` files @@ -81,7 +78,10 @@ Key characteristics: ## Quick Example ```python -from doeff import default_handlers, do, Get, Put, Tell, run +from doeff import do, run +from doeff_core_effects import Get, Put, Tell +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled @do def example_workflow(): @@ -97,11 +97,11 @@ def example_workflow(): return count -def main(): - result = run(example_workflow(), handlers=default_handlers()) - print(f"Result: {result.value}") - -main() +prog = example_workflow() +prog = writer(prog) +prog = state()(prog) +result = run(scheduled(prog)) +print(f"Result: {result}") # Result: 42 ``` ## Learning Path @@ -122,9 +122,9 @@ main() ### For Advanced Users -1. **[Advanced Effects](09-advanced-effects.md)** for Gather, Atomic -2. **[WithIntercept](22-with-intercept.md)** for cross-cutting observation -3. **[Graph Tracking](08-graph-tracking.md)** for execution visualization +1. **[Advanced Effects](09-advanced-effects.md)** for Gather, concurrency +2. **[Semaphore Effects](21-semaphore-effects.md)** for concurrency control +3. **[Effect Boundaries](17-effect-boundaries.md)** for architecture design 4. **[API Reference](13-api-reference.md)** for complete API details ## By Use Case @@ -148,7 +148,7 @@ main() - **[Basic Effects](03-basic-effects.md)** for configuration and local state - **[Async Effects](04-async-effects.md)** for external work and concurrency - **[Error Handling](05-error-handling.md)** for validation -- **[WithIntercept](22-with-intercept.md)** for observability and auditing +- **[Effect Boundaries](17-effect-boundaries.md)** for observability and auditing ### Testing @@ -157,16 +157,16 @@ main() ## Effect Quick Reference -| Category | Effects | Chapter | -|----------|---------|---------| -| **Reader** | Ask, Local | [03](03-basic-effects.md#reader-effects) | -| **State** | Get, Put, Modify, AtomicGet, AtomicUpdate | [03](03-basic-effects.md#state-effects), [09](09-advanced-effects.md#atomic-effects) | -| **Writer** | Tell, Listen, StructuredLog, slog | [03](03-basic-effects.md#writer-effects) | -| **Future** | Await, Gather | [04](04-async-effects.md) | -| **Result** | Try | [05](05-error-handling.md) | -| **Cache** | CacheGet, CachePut | [07](07-cache-system.md) | -| **Graph** | Step, Annotate, Snapshot, CaptureGraph | [08](08-graph-tracking.md) | -| **Gather** | Gather | [09](09-advanced-effects.md#gather-effects) | +| Category | Effects | Source | +|----------|---------|--------| +| **Reader** | `Ask`, `Local` | `doeff_core_effects` | +| **State** | `Get`, `Put` | `doeff_core_effects` | +| **Writer** | `Tell`, `slog`, `Listen` | `doeff_core_effects` | +| **Error** | `Try` | `doeff_core_effects` | +| **Scheduler** | `Spawn`, `Wait`, `Gather`, `Race`, `Cancel` | `doeff_core_effects.scheduler` | +| **Promise** | `CreatePromise`, `CompletePromise`, `FailPromise` | `doeff_core_effects.scheduler` | +| **Semaphore** | `CreateSemaphore`, `AcquireSemaphore`, `ReleaseSemaphore` | `doeff_core_effects.scheduler` | +| **Cache** | `MemoGet`, `CachePut` | `doeff_core_effects.cache_effects` | ## Common Patterns @@ -182,16 +182,16 @@ def application(): return result ``` -### Async + Error Handling +### Concurrency + Error Handling ```python @do -def robust_fetch(url): - result = yield Try(Await(httpx.get(url))) - if result.is_ok(): +def robust_workflow(): + result = yield Try(risky_operation()) + if isinstance(result, Ok): return result.value else: - yield Tell(f"Fetch failed: {result.error}") + yield Tell(f"Failed: {result.error}") return None ``` @@ -200,8 +200,11 @@ def robust_fetch(url): ```python @do def process_batch(items): - tasks = [process_item(item) for item in items] - results = yield Gather(*tasks) + spawned = [] + for item in items: + task = yield Spawn(process_item(item)) + spawned.append(task) + results = yield Gather(*spawned) return {"processed": len(results), "results": results} ``` diff --git a/docs/llm_unified_effects.md b/docs/llm_unified_effects.md index c3cac91ba..9e7a004ac 100644 --- a/docs/llm_unified_effects.md +++ b/docs/llm_unified_effects.md @@ -4,7 +4,7 @@ - `LLMChat` - `LLMStreamingChat` -- `LLMStructuredOutput` +- `LLMStructuredQuery` - `LLMEmbedding` ## Why @@ -20,7 +20,7 @@ Handlers inspect `effect.model`: 2. If not, it yields `Pass()` so the next outer handler can try. When a handler uses a narrow effect annotation such as `effect: LLMChat` or -`effect: LLMStructuredOutput`, the installed handler scope also carries a runtime type filter so +`effect: LLMStructuredQuery`, the installed handler scope also carries a runtime type filter so non-matching effects skip the handler entirely. This enables a single program to call multiple providers by model name. @@ -31,7 +31,7 @@ This enables a single program to call multiple providers by model name. from pydantic import BaseModel from doeff import do, run -from doeff_llm.effects import LLMChat, LLMStructuredOutput +from doeff_llm.effects import LLMChat, LLMStructuredQuery from doeff_gemini.handlers import gemini_production_handler from doeff_openai.handlers import openai_production_handler from doeff_openrouter.handlers import openrouter_production_handler @@ -44,7 +44,7 @@ class Analysis(BaseModel): @do def workflow(): - analysis = yield LLMStructuredOutput( + analysis = yield LLMStructuredQuery( messages=[{"role": "user", "content": "Analyze this code"}], response_format=Analysis, model="gpt-4o", @@ -66,11 +66,6 @@ result = run( openai_production_handler(workflow()), ), ), - env={ - "openai_api_key": "...", - "gemini_api_key": "...", - "openrouter_api_key": "...", - }, ) ``` @@ -83,3 +78,8 @@ Provider-specific effect classes remain for compatibility: - `doeff_openrouter.effects.RouterChat`, `RouterStructuredOutput` They are now deprecated aliases and emit `DeprecationWarning` when instantiated. + +## Migration Note + +- `LLMStructuredOutput` was renamed to `LLMStructuredQuery`. The old name does not exist in the current API. +- `run()` takes a single `doexpr` argument and returns the raw value. It does not accept `env=`, `handlers=`, or `trace=` keyword arguments. Environment values should be provided via handler composition (e.g. the `Ask` effect and `lazy_ask` handler). diff --git a/docs/positioning/domain-angles/ai-agents.md b/docs/positioning/domain-angles/ai-agents.md index d2ccebe8f..c6519defb 100644 --- a/docs/positioning/domain-angles/ai-agents.md +++ b/docs/positioning/domain-angles/ai-agents.md @@ -43,17 +43,17 @@ Every LLM call, tool use, and decision point is a yield. The program is a pure d ```python # Record a real agent run -result = run(research_agent("quantum computing"), handlers=[ - RecordingHandler("sessions/run_42.json"), - OpenAIHandler(api_key="..."), - ClaudeHandler(api_key="..."), - WebSearchHandler(), -]) +prog = research_agent("quantum computing") +prog = WebSearchHandler()(prog) +prog = ClaudeHandler(api_key="...")(prog) +prog = OpenAIHandler(api_key="...")(prog) +prog = RecordingHandler("sessions/run_42.json")(prog) +result = run(scheduled(prog)) # Replay to debug — zero API cost -result = run(research_agent("quantum computing"), handlers=[ - ReplayHandler("sessions/run_42.json"), -]) +prog = research_agent("quantum computing") +prog = ReplayHandler("sessions/run_42.json")(prog) +result = run(scheduled(prog)) # Instant. Deterministic. Free. ``` @@ -63,21 +63,21 @@ A user reports "the agent gave wrong results for X." Replay their session. Step ```python # Production: GPT-4o + Claude -result = run(research_agent(query), handlers=[ - OpenAIHandler(model="gpt-4o"), - ClaudeHandler(model="claude-sonnet"), -]) +prog = research_agent(query) +prog = ClaudeHandler(model="claude-sonnet")(prog) +prog = OpenAIHandler(model="gpt-4o")(prog) +result = run(scheduled(prog)) # Testing: local models -result = run(research_agent(query), handlers=[ - OllamaHandler(model="llama-3-70b"), -]) +prog = research_agent(query) +prog = OllamaHandler(model="llama-3-70b")(prog) +result = run(scheduled(prog)) # Evaluation: compare models for model in ["gpt-4o", "claude-sonnet", "gemini-2-pro"]: - result = run(research_agent(query), handlers=[ - UnifiedLLMHandler(model=model), - ]) + prog = research_agent(query) + prog = UnifiedLLMHandler(model=model)(prog) + result = run(scheduled(prog)) ``` The program never mentions a provider. The handler stack decides. @@ -98,14 +98,13 @@ def cost_cap_handler(effect: Effect, k): raise BudgetExceededError(f"${current:.2f} > ${MAX_BUDGET}") yield Delegate() -result = run( - openai_handler( - claude_handler( - cost_cap_handler(research_agent(query)), - ), - ), -) -print(result.effect_log) +prog = research_agent(query) +prog = cost_cap_handler(prog) +prog = claude_handler(prog) +prog = openai_handler(prog) +result = run(scheduled(prog)) +# The cost_cap_handler tracks costs via Get/Put effects. +# A logging handler can emit per-call stats: # LLMChat: 3 calls, $0.12 total, 4.2s latency # WebSearch: 5 calls, 2.1s latency ``` diff --git a/docs/positioning/domain-angles/batch-etl.md b/docs/positioning/domain-angles/batch-etl.md index 437431460..664741743 100644 --- a/docs/positioning/domain-angles/batch-etl.md +++ b/docs/positioning/domain-angles/batch-etl.md @@ -46,20 +46,20 @@ Retry, fallback, error handling, progress tracking, logging — all flat yields. ```python # Run with recording — captures all effects try: - result = run(process_batch(items), handlers=[ - RecordingHandler("runs/batch_2026_02_12.json"), - RealDBHandler(), - RealAPIHandler(), - ]) + prog = process_batch(items) + prog = RealAPIHandler()(prog) + prog = RealDBHandler()(prog) + prog = RecordingHandler("runs/batch_2026_02_12.json")(prog) + result = run(scheduled(prog)) except BatchError: pass # Recording captured everything up to the failure point # Resume from where it failed — replay completed effects, continue with real ones -result = run(process_batch(items), handlers=[ - ResumeHandler("runs/batch_2026_02_12.json"), # replays completed, then switches to real - RealDBHandler(), - RealAPIHandler(), -]) +prog = process_batch(items) +prog = RealAPIHandler()(prog) +prog = RealDBHandler()(prog) +prog = ResumeHandler("runs/batch_2026_02_12.json")(prog) # replays completed, then switches to real +result = run(scheduled(prog)) # Skips the 7,432 already-processed records. Continues from 7,433. ``` @@ -70,14 +70,12 @@ result = run(process_batch(items), handlers=[ ```python def test_batch_with_partial_failure(): items = ["good1", "bad1", "good2"] - result = run(process_batch(items), handlers=[ - StubDB({"good1": data1, "good2": data2}), - StubAPI({"good1": enriched1, "bad1": APIError("timeout"), "good2": enriched2}), - ]) - assert result.value.total == 3 - assert result.value.succeeded == 2 - # Verify the error was logged - assert any("bad1" in log["processed"] for log in result.writer_output) + prog = process_batch(items) + prog = StubAPI({"good1": enriched1, "bad1": APIError("timeout"), "good2": enriched2})(prog) + prog = StubDB({"good1": data1, "good2": data2})(prog) + result = run(scheduled(prog)) + assert result.total == 3 + assert result.succeeded == 2 def test_retry_behavior(): call_count = 0 @@ -88,12 +86,12 @@ def test_retry_behavior(): raise APIError("transient") return {"data": item} - result = run(process_single_item("test"), handlers=[ - FunctionHandler(fetch_data=lambda i: {"raw": i}), - FunctionHandler(enrich_from_api=flaky_api), - ]) + prog = process_single_item("test") + prog = FunctionHandler(enrich_from_api=flaky_api)(prog) + prog = FunctionHandler(fetch_data=lambda i: {"raw": i})(prog) + result = run(scheduled(prog)) assert call_count == 3 # retried twice, succeeded on third - assert result.value.data == {"data": "test"} + assert result.data == {"data": "test"} ``` No `@patch`. No `unittest.mock`. No knowledge of which module imports what. diff --git a/docs/positioning/domain-angles/fastapi.md b/docs/positioning/domain-angles/fastapi.md index 3932bac80..3b283de8a 100644 --- a/docs/positioning/domain-angles/fastapi.md +++ b/docs/positioning/domain-angles/fastapi.md @@ -70,16 +70,13 @@ def handle_request(user_id: int) -> Program[Response]: return Response(user=user, analysis=analysis) # Logging, cost tracking, retry — all as stacked handlers -result = run( - handle_request(42), - handlers=[ - logging_handler, # logs ALL effects (DB, LLM, cache) - cost_cap_handler, # caps LLM spend per request - retry_handler, # retries transient failures - db_handler, # provides real DB - openai_handler, # provides real LLM - ] -) +prog = handle_request(42) +prog = openai_handler(prog) # provides real LLM +prog = db_handler(prog) # provides real DB +prog = retry_handler(prog) # retries transient failures +prog = cost_cap_handler(prog) # caps LLM spend per request +prog = logging_handler(prog) # logs ALL effects (DB, LLM, cache) +result = run(scheduled(prog)) ``` FastAPI middleware sees HTTP requests. doeff handlers see every effect — including LLM calls, DB queries, and cache operations inside your endpoint logic. @@ -106,11 +103,11 @@ def test_endpoint(mock_llm): ```python # doeff testing: swap all handlers at once def test_handle_request(): - result = run(handle_request(42), handlers=[ - StubDB(users={42: fake_user}), - StubLLM(responses={"Analyze": "cached analysis"}), - ]) - assert result.value.analysis == "cached analysis" + prog = handle_request(42) + prog = StubLLM(responses={"Analyze": "cached analysis"})(prog) + prog = StubDB(users={42: fake_user})(prog) + result = run(scheduled(prog)) + assert result.analysis == "cached analysis" ``` No TestClient. No dependency_overrides. No @patch. One line per fake service. @@ -123,30 +120,27 @@ doeff doesn't require rewriting your FastAPI app. You can adopt it incrementally # Step 1: Use doeff inside a single endpoint @app.post("/analyze") async def analyze(request: AnalyzeRequest): - result = run( - analyze_pipeline(request.text), - handlers=[openai_handler, cache_handler, db_handler], - ) - return result.value + prog = analyze_pipeline(request.text) + prog = db_handler(prog) + prog = cache_handler(prog) + prog = openai_handler(prog) + return run(scheduled(prog)) # Step 2: Add recording for debugging @app.post("/analyze") async def analyze(request: AnalyzeRequest): - result = run( - analyze_pipeline(request.text), - handlers=[ - RecordingHandler(f"traces/{request.id}.json"), - openai_handler, cache_handler, db_handler, - ], - ) - return result.value + prog = analyze_pipeline(request.text) + prog = db_handler(prog) + prog = cache_handler(prog) + prog = openai_handler(prog) + prog = RecordingHandler(f"traces/{request.id}.json")(prog) + return run(scheduled(prog)) # Step 3: Replay production issues # When a user reports a bug, replay their trace locally -result = run( - analyze_pipeline("the problematic input"), - handlers=[ReplayHandler("traces/bug_report_123.json")], -) +prog = analyze_pipeline("the problematic input") +prog = ReplayHandler("traces/bug_report_123.json")(prog) +result = run(scheduled(prog)) ``` ## The Pitch diff --git a/docs/positioning/domain-angles/gradio-prototyping.md b/docs/positioning/domain-angles/gradio-prototyping.md index f27626801..0ce037be1 100644 --- a/docs/positioning/domain-angles/gradio-prototyping.md +++ b/docs/positioning/domain-angles/gradio-prototyping.md @@ -115,11 +115,11 @@ The user runs the pipeline, sees the result, and thinks "what if I used factor=2 ```python # Replay Denoise from recording, override Upscale params, continue live -result = run(image_pipeline(image), handlers=[ - ReplayHandler("trace.json", replay_until="Upscale"), # instant - OverrideHandler(Upscale, {"factor": 2}), # modify - RealHandler(), # continue -]) +prog = image_pipeline(image) +prog = RealHandler()(prog) # continue +prog = OverrideHandler(Upscale, {"factor": 2})(prog) # modify +prog = ReplayHandler("trace.json", replay_until="Upscale")(prog) # instant +result = run(scheduled(prog)) ``` In the Gradio UI: @@ -147,16 +147,17 @@ The recorded effect trace isn't just a log — it's the full execution with all ```python # Researcher A runs the pipeline, records everything -result = run(style_transfer(content, style), handlers=[ - RecordingHandler("experiments/run_42.json"), - *gpu_handlers, -]) +prog = style_transfer(content, style) +for h in reversed(gpu_handlers): + prog = h(prog) +prog = RecordingHandler("experiments/run_42.json")(prog) +result = run(scheduled(prog)) # run_42.json contains: every effect, every intermediate tensor, every parameter # Researcher B opens the trace in their own Gradio app — no GPU needed -result = run(style_transfer(content, style), handlers=[ - InteractiveReplayHandler("experiments/run_42.json"), -]) +prog = style_transfer(content, style) +prog = InteractiveReplayHandler("experiments/run_42.json")(prog) +result = run(scheduled(prog)) # They can browse every intermediate result, fork, modify parameters ``` @@ -168,15 +169,16 @@ This one is simple but high-value for the Gradio community: ```python # Before the conference: record your demo session -run(demo_pipeline(inputs), handlers=[ - RecordingHandler("demo_session.json"), - *real_handlers, -]) +prog = demo_pipeline(inputs) +for h in reversed(real_handlers): + prog = h(prog) +prog = RecordingHandler("demo_session.json")(prog) +run(scheduled(prog)) # At the conference: replay mode -run(demo_pipeline(inputs), handlers=[ - ReplayHandler("demo_session.json"), -]) +prog = demo_pipeline(inputs) +prog = ReplayHandler("demo_session.json")(prog) +run(scheduled(prog)) # Works on airplane WiFi. No API key. No GPU. Same outputs. ``` diff --git a/docs/positioning/domain-angles/ml-infrastructure.md b/docs/positioning/domain-angles/ml-infrastructure.md index 1ae7de340..ec31076c2 100644 --- a/docs/positioning/domain-angles/ml-infrastructure.md +++ b/docs/positioning/domain-angles/ml-infrastructure.md @@ -276,13 +276,10 @@ def retry_handler(max_retries=3, backoff=1.0, transient_errors=None): return handler # Use it -result = run( - execute_kubectl("get pods"), - handlers=[ - retry_handler(max_retries=3, backoff=1.0), - system_call_handler(), - ] -) +prog = execute_kubectl("get pods") +prog = system_call_handler()(prog) +prog = retry_handler(max_retries=3, backoff=1.0)(prog) +result = run(scheduled(prog)) ``` **Retry logic written once, applies to all SystemCall effects.** Want circuit breaker? Add a circuit breaker handler. Want jitter? Modify the retry handler once. The business logic (`execute_kubectl`, `krsync_to_pod`) never changes. @@ -354,15 +351,14 @@ def logging_handler(): return handler # Silent mode: just don't include the logging handler -result_verbose = run( - execute_kubectl("get pods"), - handlers=[logging_handler(), system_call_handler()] -) - -result_silent = run( - execute_kubectl("get pods"), - handlers=[system_call_handler()] # no logging handler -) +prog = execute_kubectl("get pods") +prog = system_call_handler()(prog) +prog = logging_handler()(prog) +result_verbose = run(scheduled(prog)) + +prog = execute_kubectl("get pods") +prog = system_call_handler()(prog) # no logging handler +result_silent = run(scheduled(prog)) ``` **One function, not two.** Logging is composed through handlers. Polling loops use the silent handler stack. User-facing commands use the verbose stack. The business logic never changes. @@ -425,20 +421,17 @@ def test_job_deployment(): return handler # Run with stub handlers - result = run( - persistent_ml_platform_job_core( - job_spec=test_job_spec, - mount_plan=test_mount_plan, - ), - handlers=[ - stub_system_call_handler(), - stub_sync_handler(), - ] + prog = persistent_ml_platform_job_core( + job_spec=test_job_spec, + mount_plan=test_mount_plan, ) + prog = stub_sync_handler()(prog) + prog = stub_system_call_handler()(prog) + result = run(scheduled(prog)) # Verify orchestration logic - assert result.value.status == "completed" - assert result.value.pod_name == "test-job" + assert result.status == "completed" + assert result.pod_name == "test-job" def test_retry_on_transient_failure(): """Test retry logic without real infrastructure.""" @@ -456,16 +449,13 @@ def test_retry_on_transient_failure(): yield Delegate() return handler - result = run( - execute_kubectl("get pods"), - handlers=[ - retry_handler(max_retries=3), - flaky_system_call_handler(), - ] - ) + prog = execute_kubectl("get pods") + prog = flaky_system_call_handler()(prog) + prog = retry_handler(max_retries=3)(prog) + result = run(scheduled(prog)) assert call_count == 3 # retried twice, succeeded on third - assert result.value == "success" + assert result == "success" ``` **Tests are fast, deterministic, and require no infrastructure.** The orchestration logic is unit-testable. Integration tests still exist (with real handlers), but unit tests cover the majority of logic. @@ -748,20 +738,24 @@ def vertex_ai_handlers(registry: str, gcs_bucket: str): # ---- Usage: same function, different handlers ---- # Local Docker -result = run(run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts), - handlers=[local_docker_handlers(), retry_handler(), logging_handler()]) +prog = run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts) +prog = logging_handler()(retry_handler()(local_docker_handlers()(prog))) +result = run(scheduled(prog)) # K8s GPUaaS -result = run(run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts), - handlers=[k8s_gpuaas_handlers(registry), retry_handler(), logging_handler()]) +prog = run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts) +prog = logging_handler()(retry_handler()(k8s_gpuaas_handlers(registry)(prog))) +result = run(scheduled(prog)) # Vertex AI -result = run(run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts), - handlers=[vertex_ai_handlers(registry, bucket), retry_handler(), logging_handler()]) +prog = run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts) +prog = logging_handler()(retry_handler()(vertex_ai_handlers(registry, bucket)(prog))) +result = run(scheduled(prog)) # Test (no infrastructure) -result = run(run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts), - handlers=[stub_handlers()]) +prog = run_ml_job(project, schematic, "python train.py", A100_CONFIG, mounts) +prog = stub_handlers()(prog) +result = run(scheduled(prog)) ``` The key insight visualized: @@ -797,21 +791,19 @@ And because handlers compose, cross-cutting concerns stack orthogonally: ```python # Production K8s with retry + logging + event bus + recording -result = run(run_ml_job(project, schematic, script, config, mounts), - handlers=[ - recording_handler("runs/run_042.json"), # audit trail - retry_handler(max_retries=3), # transient error recovery - event_bus_handler(bus), # observability - logging_handler(), # user-facing logs - k8s_gpuaas_handlers(registry), # K8s-specific operations - ]) +prog = run_ml_job(project, schematic, script, config, mounts) +prog = k8s_gpuaas_handlers(registry)(prog) # K8s-specific operations +prog = logging_handler()(prog) # user-facing logs +prog = event_bus_handler(bus)(prog) # observability +prog = retry_handler(max_retries=3)(prog) # transient error recovery +prog = recording_handler("runs/run_042.json")(prog) # audit trail +result = run(scheduled(prog)) # Same job, replay a failed run on Vertex AI instead -result = run(run_ml_job(project, schematic, script, config, mounts), - handlers=[ - replay_handler("runs/run_042.json"), # replay decisions from failed run - vertex_ai_handlers(registry, bucket), # but on Vertex AI this time - ]) +prog = run_ml_job(project, schematic, script, config, mounts) +prog = vertex_ai_handlers(registry, bucket)(prog) # but on Vertex AI this time +prog = replay_handler("runs/run_042.json")(prog) # replay decisions from failed run +result = run(scheduled(prog)) ``` **You can't do this with DI.** DI can swap `KrsyncService` for `GsutilService`. But DI cannot replay a K8s run on Vertex AI by composing a replay handler with a different environment handler. That requires controlling execution flow — which is what effects do. @@ -937,13 +929,10 @@ def event_bus_handler(bus): return handler # Use it -result = run( - execute_kubectl("get pods"), - handlers=[ - event_bus_handler(ml_nexus_event_bus), - system_call_handler(), - ] -) +prog = execute_kubectl("get pods") +prog = system_call_handler()(prog) +prog = event_bus_handler(ml_nexus_event_bus)(prog) +result = run(scheduled(prog)) ``` **Event emission is automatic.** Every `SystemCall` effect triggers events through the handler. The business logic never mentions the event bus. diff --git a/docs/positioning/domain-angles/ml-pipelines.md b/docs/positioning/domain-angles/ml-pipelines.md index 739806a22..b97adb11c 100644 --- a/docs/positioning/domain-angles/ml-pipelines.md +++ b/docs/positioning/domain-angles/ml-pipelines.md @@ -34,22 +34,22 @@ Injected.mzip(a, b, c) -> a, b, c = yield Gather(prog_a, prog_b, ### Record a training experiment: ```python -result = run(train_experiment(), handlers=[ - RecordingHandler("runs/exp_2026_02_12.json"), - RealGPUHandler(), - WandBHandler(), - OpenAIHandler(), -]) +prog = train_experiment() +prog = OpenAIHandler()(prog) +prog = WandBHandler()(prog) +prog = RealGPUHandler()(prog) +prog = RecordingHandler("runs/exp_2026_02_12.json")(prog) +result = run(scheduled(prog)) # Cost: $50 in API calls, 4 hours of GPU time ``` ### Replay to re-analyze with different metrics ($0, instant): ```python -result = run(evaluate_model(), handlers=[ - ReplayHandler("runs/exp_2026_02_12.json"), # replays all IO - NewMetricsHandler(), # computes new metrics -]) +prog = evaluate_model() +prog = NewMetricsHandler()(prog) # computes new metrics +prog = ReplayHandler("runs/exp_2026_02_12.json")(prog) # replays all IO +result = run(scheduled(prog)) # Cost: $0. No GPU. No API calls. Instant. ``` @@ -57,10 +57,10 @@ result = run(evaluate_model(), handlers=[ ```python for lr in [0.001, 0.01, 0.1]: - result = run(train_experiment(), handlers=[ - ReplayHandler("runs/exp_2026_02_12.json"), - OverrideHandler({"learning_rate": lr}), - ]) + prog = train_experiment() + prog = OverrideHandler({"learning_rate": lr})(prog) + prog = ReplayHandler("runs/exp_2026_02_12.json")(prog) + result = run(scheduled(prog)) # "This saved us $4,950 in API costs" ``` diff --git a/docs/positioning/domain-angles/trading-backtesting.md b/docs/positioning/domain-angles/trading-backtesting.md index 7967fdaae..7f6a592d4 100644 --- a/docs/positioning/domain-angles/trading-backtesting.md +++ b/docs/positioning/domain-angles/trading-backtesting.md @@ -129,16 +129,12 @@ def simulated_time_handler(start_time): return handler # Backtest: 10 years of minute data -> runs in seconds -result = run( - mean_reversion_strategy("AAPL", window=20), - handlers=[ - backtest_market_handler(load_csv("AAPL_2015_2025.csv")), - simulated_time_handler(start_time=datetime(2015, 1, 1).timestamp()), - ], - store={"portfolio": {}, "cash": 100_000}, -) - -# result.writer_output contains every price check, every trade signal +prog = mean_reversion_strategy("AAPL", window=20) +prog = simulated_time_handler(start_time=datetime(2015, 1, 1).timestamp())(prog) +prog = backtest_market_handler(load_csv("AAPL_2015_2025.csv"))(prog) +result = run(scheduled(prog)) + +# Every price check and trade signal is visible through the effect trace # Instant. Deterministic. Reproducible. ``` @@ -181,14 +177,10 @@ def real_time_handler(): return handler # Paper trading: real prices, simulated orders, real time -result = run( - mean_reversion_strategy("AAPL", window=20), - handlers=[ - live_market_handler(), - real_time_handler(), - ], - store={"portfolio": {}, "cash": 100_000}, -) +prog = mean_reversion_strategy("AAPL", window=20) +prog = real_time_handler()(prog) +prog = live_market_handler()(prog) +result = run(scheduled(prog)) ``` ## Handler Mode 3: Live Trading (Real Everything) @@ -221,27 +213,21 @@ def live_execution_handler(broker_client): return handler # Live trading: real everything, with recording for audit -result = run( - mean_reversion_strategy("AAPL", window=20), - handlers=[ - RecordingHandler("trades/live_2026_02_12.json"), # audit trail - live_execution_handler(alpaca_client), - real_time_handler(), - ], -) +prog = mean_reversion_strategy("AAPL", window=20) +prog = real_time_handler()(prog) +prog = live_execution_handler(alpaca_client)(prog) +prog = RecordingHandler("trades/live_2026_02_12.json")(prog) # audit trail +result = run(scheduled(prog)) ``` ## Handler Mode 4: Replay a Live Session as Backtest ```python # Yesterday's live session, replayed instantly for analysis -result = run( - mean_reversion_strategy("AAPL", window=20), - handlers=[ - ReplayHandler("trades/live_2026_02_11.json"), - simulated_time_handler(start_time=yesterday_start), - ], -) +prog = mean_reversion_strategy("AAPL", window=20) +prog = simulated_time_handler(start_time=yesterday_start)(prog) +prog = ReplayHandler("trades/live_2026_02_11.json")(prog) +result = run(scheduled(prog)) # Same decisions, same prices, but runs in milliseconds instead of hours. # Useful for: analyzing why a trade was made, verifying strategy logic, # generating reports from recorded data. @@ -253,23 +239,15 @@ result = run( # "What if I had used a 50-period SMA instead of 20?" # Replay market data but let the strategy re-decide -result_20 = run( - mean_reversion_strategy("AAPL", window=20), - handlers=[ - backtest_market_handler(historical_data), - simulated_time_handler(start), - ], - store={"portfolio": {}, "cash": 100_000}, -) - -result_50 = run( - mean_reversion_strategy("AAPL", window=50), # only this changed - handlers=[ - backtest_market_handler(historical_data), # same data - simulated_time_handler(start), - ], - store={"portfolio": {}, "cash": 100_000}, -) +prog = mean_reversion_strategy("AAPL", window=20) +prog = simulated_time_handler(start)(prog) +prog = backtest_market_handler(historical_data)(prog) +result_20 = run(scheduled(prog)) + +prog = mean_reversion_strategy("AAPL", window=50) # only this changed +prog = simulated_time_handler(start)(prog) +prog = backtest_market_handler(historical_data)(prog) # same data +result_50 = run(scheduled(prog)) # Compare P&L, trade count, drawdown — same market, different parameters ``` @@ -315,14 +293,11 @@ With effects, each handler manages its own concern: Combine with the [Gradio positioning](gradio-prototyping.md) for a live trading dashboard: ```python -result = run( - mean_reversion_strategy("AAPL"), - handlers=[ - gradio_streaming_handler, # live chart, trade log, P&L curve - live_market_handler(), - real_time_handler(), - ], -) +prog = mean_reversion_strategy("AAPL") +prog = real_time_handler()(prog) +prog = live_market_handler()(prog) +prog = gradio_streaming_handler(prog) # live chart, trade log, P&L curve +result = run(scheduled(prog)) ``` Every `GetPrice`, `PlaceOrder`, and `Tell` pushes to the Gradio UI in real-time. The strategy doesn't know about Gradio. The dashboard is an orthogonal handler. diff --git a/docs/positioning/langchain-critique.md b/docs/positioning/langchain-critique.md index abc7e9d17..bc4242f7c 100644 --- a/docs/positioning/langchain-critique.md +++ b/docs/positioning/langchain-critique.md @@ -107,14 +107,11 @@ def test_rag_chain(mock_parse, mock_predict, mock_retrieve): **doeff approach:** ```python def test_rag_pipeline(): - result = run( - rag_pipeline("query"), - handlers=[ - StubLLM({"Analyze": "cached analysis"}), - InMemoryRetriever({"query": [doc1, doc2]}), - ] - ) - assert result.value == expected + prog = rag_pipeline("query") + prog = InMemoryRetriever({"query": [doc1, doc2]})(prog) + prog = StubLLM({"Analyze": "cached analysis"})(prog) + result = run(scheduled(prog)) + assert result == expected ``` No patching. No mock objects. No knowledge of implementation internals. Swap the handler, the program doesn't change. diff --git a/docs/positioning/three-stage-pitch.md b/docs/positioning/three-stage-pitch.md index 2deb90555..9a5f51cf2 100644 --- a/docs/positioning/three-stage-pitch.md +++ b/docs/positioning/three-stage-pitch.md @@ -112,12 +112,12 @@ def test_process_order(mock_sleep, mock_http, mock_db, mock_cache_set, mock_cach ```python def test_process_order(): - result = run(process_order("order-123"), handlers=[ - InMemoryCache(), - StubDB({"order-123": Order(qty=5, product_id="X")}), - StubHTTP({"/api/price/X": {"price": 100}}), - ]) - assert result.value == 500 + prog = process_order("order-123") + prog = StubHTTP({"/api/price/X": {"price": 100}})(prog) + prog = StubDB({"order-123": Order(qty=5, product_id="X")})(prog) + prog = InMemoryCache()(prog) + result = run(scheduled(prog)) + assert result == 500 ``` **What the audience sees:** @@ -135,20 +135,20 @@ def test_process_order(): ### Record a production run: ```python -result = run(process_order("order-123"), handlers=[ - RecordingHandler("runs/order-123-20260212.json"), - RealDB(), - RealHTTP(), - RealCache(), -]) +prog = process_order("order-123") +prog = RealCache()(prog) +prog = RealHTTP()(prog) +prog = RealDB()(prog) +prog = RecordingHandler("runs/order-123-20260212.json")(prog) +result = run(scheduled(prog)) ``` ### Replay without any external services: ```python -result = run(process_order("order-123"), handlers=[ - ReplayHandler("runs/order-123-20260212.json"), -]) +prog = process_order("order-123") +prog = ReplayHandler("runs/order-123-20260212.json")(prog) +result = run(scheduled(prog)) # Zero network calls. Zero DB queries. Instant. Deterministic. ``` @@ -156,10 +156,10 @@ result = run(process_order("order-123"), handlers=[ ```python # Same recorded data, but with a new pricing algorithm -result = run(process_order("order-123"), handlers=[ - ReplayHandler("runs/order-123-20260212.json"), - NewPricingHandler(), # Override just the pricing effect -]) +prog = process_order("order-123") +prog = NewPricingHandler()(prog) # Override just the pricing effect +prog = ReplayHandler("runs/order-123-20260212.json")(prog) +result = run(scheduled(prog)) ``` **What the audience sees:** diff --git a/docs/program-architecture-overview.md b/docs/program-architecture-overview.md index 2bb7bcb47..3f707aaa3 100644 --- a/docs/program-architecture-overview.md +++ b/docs/program-architecture-overview.md @@ -69,24 +69,23 @@ h0(h1(h2(program))) ## Rust VM Stepping Engine The step engine is a mode/state machine that repeatedly executes one transition at a time. -The same engine is used by both `run` and `async_run`. ```mermaid flowchart TD - A[run / async_run] --> B{Input kind} + A[run] --> B{Input kind} B -->|DoExpr| C[Use as root control node] - B -->|EffectValue| D[Normalize to Perform(effect)] + B -->|EffectValue| D[Normalize to Perform] D --> C C --> E[Install handler stack from handler installers] E --> F[VM step loop] F --> G{Yield classification} G -->|DoCtrl| H[Evaluate DoCtrl] - G -->|EffectValue| I[Normalize to Perform(effect)] + G -->|EffectValue| I[Normalize to Perform] I --> J[Dispatch through handler stack] H --> K{DoCtrl variant} - K -->|Perform(effect)| J + K -->|Perform effect| J K -->|Other control node| L[Update VM mode/state] J --> M{Handler action} @@ -98,7 +97,7 @@ flowchart TD O -->|Continue| F O -->|NeedsPython| P[Driver executes PythonCall and feeds result] P --> F - O -->|Done / Failed| Q[Build RunResult] + O -->|Done / Failed| Q[Return raw value or raise] ``` ## Call Stack Tracking @@ -112,35 +111,31 @@ that metadata attached. `Frame::PythonGenerator` entries. This keeps stack reconstruction deterministic in VM state while Python object access remains in the driver. -## Effect Observation (`trace=True`) +## Effect Observation (`WithObserve`) -`run(..., trace=True)` and `async_run(..., trace=True)` enable VM step tracing in `RunResult.trace`. -Each entry records step-level execution state: - -- `step` -- `event` -- `mode` -- `pending` -- `dispatch_depth` -- `result` +Effect observation is done by wrapping a program with `WithObserve(observer, body)`. +The observer callback receives each effect dispatched within the body. There is no `trace=True` +parameter on `run()` — observation is composed into the program like any other handler. Example: ```python -result = run(program, handlers=default_handlers(), trace=True) -first = result.trace[0] -# {'step': 1, 'event': 'enter', 'mode': 'Deliver', ...} -``` +from doeff import run, WithObserve -This trace shows when execution enters `Perform` dispatch and how handler-dispatch depth changes -during stepping. +observations = [] -## Async Boundary +def my_observer(effect): + observations.append(effect) -`PythonAsyncSyntaxEscape` is the VM escape for Python async integration. +result = run(WithObserve(my_observer, program)) +# observations now contains each effect that was dispatched +``` + +## Async Boundary -- `run(...)`: synchronous stepping path. -- `async_run(...)`: same step semantics plus async escape/await/resume handling. +For async programs, use `run(scheduled(program))` where `scheduled` comes from +`doeff_core_effects.scheduler`. The `scheduled` handler enables cooperative scheduling +of async tasks within the VM's synchronous step loop. ## Summary @@ -148,4 +143,7 @@ during stepping. - Control and effect payloads are separated. - `Perform(effect)` is the sole dispatch boundary. - Handlers are a nested stack with deterministic inner-to-outer dispatch. -- Rust VM stepping is the execution core for both sync and async runners. +- Rust VM stepping is the execution core. +- `run(doexpr)` takes a single argument and returns the raw value (or raises on error). +- Observation uses `WithObserve(observer, body)`, not a trace parameter. +- Async execution uses `run(scheduled(...))`. diff --git a/docs/proposals/001-doeff-flow-webui.md b/docs/proposals/001-doeff-flow-webui.md index a947a4bbf..826f8131c 100644 --- a/docs/proposals/001-doeff-flow-webui.md +++ b/docs/proposals/001-doeff-flow-webui.md @@ -1,5 +1,16 @@ # Proposal: doeff-flow — Node-Based WebUI for doeff +> **Note**: This document reflects the state at time of writing (2025-01-29) and some APIs have since changed. Specifically: +> - `.intercept()` was removed — use `WithObserve(observer, body)` for effect observation +> - `EffectCallTree`, `EffectObservation`, `EffectCreationContext` were removed +> - `WGraph`, `WStep`, `WNode` were removed +> - `graph_snapshot` / `graph_to_html` were removed +> - `AsyncRuntime` / `SyncRuntime` / `SimulationRuntime` are not real doeff classes — use `run(doexpr)` (single argument, returns raw value) and `run(scheduled(...))` for async +> - `RunResult` was removed — `run()` returns the raw value directly +> - `default_handlers()` was removed — compose handlers individually +> +> The architectural vision and comparison with ComfyUI remain relevant. Implementation would need to use the current API surface. + **Status**: Draft **Author**: Design discussion (2025-01-29) **Package**: `doeff-flow` (new subpackage) @@ -25,10 +36,9 @@ ComfyUI demonstrates the value of node-based workflow editors for AI pipelines, doeff already has the primitives for a cleaner solution: - `Program[T]` — lazy, composable computations - `Effect` — data describing operations -- `WGraph`, `WStep`, `WNode` — DAG representation +- `WithObserve` — effect observation - `@cache` — memoization - `Gather` — parallel execution -- Effect interception — event emission ## Design @@ -46,23 +56,23 @@ The missing piece is **metadata** for the UI to know what widgets to render. ### Architecture ``` -┌─────────────────────────────────────────────────────────────────────┐ -│ Hy Source │ -│ (defnode ...) (defgraph ...) ──── single source of truth │ -└───────────────────────────────┬─────────────────────────────────────┘ - │ - ┌───────────┴───────────┐ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ UI Schema │ │ doeff Program │ - │ (JSON) │ │ (runtime) │ - └────────┬────────┘ └────────┬────────┘ - │ │ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ Frontend │◄────│ Runtime │ - │ (litegraph) │ SSE │ (async exec) │ - └─────────────────┘ └─────────────────┘ ++-----------------------------------------------------------------+ +| Hy Source | +| (defnode ...) (defgraph ...) --- single source of truth | ++-------------------------------+---------------------------------+ + | + +-----------+-----------+ + v v + +-----------------+ +-----------------+ + | UI Schema | | doeff Program | + | (JSON) | | (runtime) | + +--------+--------+ +--------+--------+ + | | + v v + +-----------------+ +-----------------+ + | Frontend |<----| Runtime | + | (litegraph) | SSE | (async exec) | + +-----------------+ +-----------------+ ``` ### Hy Macros @@ -146,26 +156,27 @@ The runtime emits events during execution: {:type :graph-done :output-node "image" :total-ms 3420} ``` -Using doeff's effect interception: +Using doeff's `WithObserve`: -```hy -(defn with-events [program event-sink] - (.intercept program - (fn [effect] - (put! event-sink {:type :effect - :effect-type (type effect) - :node (current-node)}) - effect))) +```python +from doeff import WithObserve + +def with_events(program, event_sink): + def observer(effect): + event_sink.put({"type": "effect", + "effect_type": type(effect).__name__, + "node": current_node()}) + return WithObserve(observer, program) ``` -### Round-Trip: UI ↔ Code +### Round-Trip: UI <-> Code The system supports bidirectional conversion: ``` -Hy Code ──compile──▶ JSON Schema ──render──▶ Web UI - ▲ │ - └────────────serialize────────────────────────┘ +Hy Code --compile--> JSON Schema --render--> Web UI + ^ | + +------------serialize-----------------------+ ``` UI edits can be serialized back to Hy code for version control. @@ -183,10 +194,10 @@ UI edits can be serialized back to Hy code for version control. | **Composability** | None | Full (just functions) | | **Testing** | Requires GPU + models | **Mock handlers, instant** | | **Caching** | Custom hash logic | doeff `@cache` decorator | -| **Events** | Bespoke WebSocket | Effect interception | +| **Events** | Bespoke WebSocket | `WithObserve` | | **Error handling** | Try/catch scattered | `Try` effect, structured | | **Extensibility** | Monkey-patch nodes | Compose programs | -| **Round-trip** | UI → JSON only | UI ↔ Hy ↔ JSON | +| **Round-trip** | UI -> JSON only | UI <-> Hy <-> JSON | ## Key Architectural Advantage: Complete Mockability @@ -201,63 +212,52 @@ UI edits can be serialized back to Hy code for version control. ```python # Test workflow with mock handlers - NO GPU, NO models, milliseconds -mock_handlers = { - ImageLoad: lambda e, ts, s: ContinueValue(FakeImage(512, 512), ...), - Encode: lambda e, ts, s: ContinueValue(FakeEmbedding(), ...), - Sample: lambda e, ts, s: ContinueValue(FakeLatent(), ...), - Decode: lambda e, ts, s: ContinueValue(FakeImage(1024, 1024), ...), -} +from doeff import do, run +from doeff.program import handler + +@do +def mock_image_load_handler(effect, k): + from doeff.program import Resume + yield Resume(k, FakeImage(512, 512)) + +@do +def mock_encode_handler(effect, k): + from doeff.program import Resume + yield Resume(k, FakeEmbedding()) -runtime = AsyncRuntime(handlers=mock_handlers) @pytest.mark.asyncio async def test_workflow_executes_correct_order(): """Test graph topology - NO GPU needed.""" workflow = load_workflow("txt2img.hy") executed = [] - - # Track which effects run - tracking_runtime = AsyncRuntime(handlers=make_tracking_handlers(executed)) - await tracking_runtime.run(workflow) - - assert executed == ["ImageLoad", "Encode", "Sample", "Decode"] -@pytest.mark.asyncio -async def test_caching_skips_unchanged_nodes(): - """Test cache behavior - NO GPU needed.""" - workflow = load_workflow("txt2img.hy") - call_counts = Counter() - - counting_runtime = AsyncRuntime(handlers=make_counting_handlers(call_counts)) - await counting_runtime.run(workflow) - await counting_runtime.run(workflow) # Second run - - assert call_counts["Encode"] == 1 # Cached on second run + # Track which effects run via WithObserve + from doeff import WithObserve + def tracker(effect): + executed.append(type(effect).__name__) -@pytest.mark.asyncio -async def test_error_propagation(): - """Test error handling - NO GPU needed.""" - def failing_sample(e, ts, s): - raise RuntimeError("Out of memory") - - runtime = AsyncRuntime(handlers={**mock_handlers, Sample: failing_sample}) - result = await runtime.run(workflow) - - assert result.is_err() - assert "Out of memory" in str(result.error) + tracked = WithObserve(tracker, workflow) + result = run( + handler(mock_image_load_handler)( + handler(mock_encode_handler)(tracked) + ) + ) + + assert executed == ["ImageLoad", "Encode", "Sample", "Decode"] ``` ### Testing Capability Comparison | Capability | ComfyUI | doeff-flow | |------------|---------|------------| -| Test workflow logic | ❌ Requires models | ✅ Mock handlers | -| Test node ordering | ❌ Requires execution | ✅ Inspect Program | -| Test caching | ❌ Run twice with GPU | ✅ Mock + count | -| Test error handling | ❌ Force real errors | ✅ Inject failures | -| Run in CI | ❌ Need GPU runner | ✅ Any CI | -| Test speed | ❌ Minutes | ✅ Milliseconds | -| Test parallelism | ❌ Real Gather | ✅ Mock Gather | +| Test workflow logic | No (requires models) | Yes (mock handlers) | +| Test node ordering | No (requires execution) | Yes (inspect Program) | +| Test caching | No (run twice with GPU) | Yes (mock + count) | +| Test error handling | No (force real errors) | Yes (inject failures) | +| Run in CI | No (need GPU runner) | Yes (any CI) | +| Test speed | No (minutes) | Yes (milliseconds) | +| Test parallelism | No (real Gather) | Yes (mock Gather) | This is a **fundamental architectural advantage** — workflows created in the WebUI can be tested via CLI with mock handlers, enabling TDD for AI pipelines. @@ -313,32 +313,20 @@ doeff-flow has full language control flow: ### 4. Debugging & Observability -doeff provides rich introspection out of the box: - -| Feature | What It Does | -|---------|--------------| -| `EffectCallTree` | Hierarchical view of which `@do` functions produced which effects | -| `EffectObservation` | Record of every effect executed | -| `EffectCreationContext` | File, line, function where effect was created | -| `RuntimeResult` | Full execution trace, logs, state, graph | -| `WGraph` visualization | vis.js HTML output of computation graph | +doeff provides observation out of the box via `WithObserve(observer, body)`: ```python -result = await runtime.run(workflow) - -# See exactly what happened -print(result.call_tree.visualize_ascii()) -# └─ txt2img() -# ├─ load_model('sdxl') -# │ └─ LoadModel('sdxl') -# ├─ encode('a cat') -# │ └─ Encode(...) -# └─ sample(...) -# └─ Sample(...) x20 - -# Where did the error originate? -print(result.error.creation_context) -# File "workflow.hy", line 42, in sample +from doeff import run, WithObserve + +observations = [] +def observer(effect): + observations.append({"type": type(effect).__name__, "effect": effect}) + +result = run(WithObserve(observer, workflow)) + +# See exactly what effects were dispatched +for obs in observations: + print(obs) ``` ComfyUI: Console logs. Good luck tracing errors. @@ -349,14 +337,17 @@ Since effects are data, executions can be recorded and replayed: ```python # Record execution -result = await runtime.run(workflow) -recorded_effects = result.effect_observations +from doeff import run, WithObserve -# Replay with same effect results — NO GPU! -replay_handlers = make_replay_handlers(recorded_effects) -replayed = await AsyncRuntime(handlers=replay_handlers).run(workflow) +recorded = [] +def recorder(effect): + recorded.append(effect) -assert replayed.value == result.value # Deterministic! +result = run(WithObserve(recorder, workflow)) + +# Replay with pre-recorded effect results — NO GPU! +replay_handler = make_replay_handler(recorded) +replayed = run(replay_handler(workflow)) ``` Use cases: @@ -364,61 +355,40 @@ Use cases: - **Time-travel debugging** — step through effect by effect - **Golden master testing** — record once, replay forever -### 6. Multiple Runtimes - -| Runtime | Use Case | -|---------|----------| -| `AsyncRuntime` | Production with real GPU | -| `SyncRuntime` | Simple scripts, blocking | -| `SimulationRuntime` | Test with fake/controlled time | +### 6. Execution Model ```python -# Test timeout behavior without actually waiting -sim_runtime = SimulationRuntime() - -async def test_timeout(): - workflow = with_timeout(long_workflow, seconds=30) - sim_runtime.advance_time(31) # Instant! - - result = await sim_runtime.run(workflow) - assert result.is_err() - assert "timeout" in str(result.error) -``` +from doeff import run +from doeff_core_effects.scheduler import scheduled -ComfyUI: One runtime. Real time. Real waiting. +# Synchronous +result = run(handler_stack(workflow)) -### 7. Effect Interception (Middleware) +# With async scheduling +result = run(scheduled(handler_stack(workflow))) +``` + +### 7. Effect Observation (Middleware) Add cross-cutting concerns without modifying nodes: -```hy -;; Add logging to every effect -(defn with-logging [program] - (.intercept program - (fn [effect] - (print f"Executing: {effect}") - effect))) - -;; Add metrics collection -(defn with-metrics [program metrics-client] - (.intercept program - (fn [effect] - (.increment metrics-client (type effect)) - effect))) - -;; Add retry logic to all IO effects -(defn with-retry [program] - (.intercept program - (fn [effect] - (if (io-effect? effect) - (retry-effect effect :attempts 3) - effect)))) - -;; Compose middleware -(-> workflow - (with-logging) - (with-metrics prometheus) - (with-retry)) +```python +from doeff import WithObserve + +# Add logging to every effect +def with_logging(program): + def log_observer(effect): + print(f"Executing: {effect}") + return WithObserve(log_observer, program) + +# Add metrics collection +def with_metrics(program, metrics_client): + def metrics_observer(effect): + metrics_client.increment(type(effect).__name__) + return WithObserve(metrics_observer, program) + +# Compose middleware +result = run(with_logging(with_metrics(workflow, prometheus))) ``` ComfyUI: No middleware. Every node handles its own concerns. @@ -433,15 +403,10 @@ Clean configuration via the `Ask` effect: api-key (yield (Ask "api_key")) cache-dir (yield (Ask "cache_dir"))] ...)) - -;; Inject different configs per environment -(runtime.run workflow :env {"model_path" "/prod/models" - "api_key" (get-secret "PROD_KEY")}) - -(runtime.run workflow :env {"model_path" "/test/models" - "api_key" "test-key"}) ``` +Environment values are provided via handler composition (e.g. `lazy_ask` handler). + ComfyUI: Hardcoded paths or global configuration. ### 9. Incremental Execution @@ -449,13 +414,13 @@ ComfyUI: Hardcoded paths or global configuration. Like a build system (Make, Bazel) — only re-run what changed: ```python -result1 = await runtime.run(workflow, inputs={"prompt": "a cat"}) -# Executes: load_model → encode → sample → decode +result1 = run(handler_stack(workflow_with_prompt("a cat"))) +# Executes: load_model -> encode -> sample -> decode -result2 = await runtime.run(workflow, inputs={"prompt": "a dog"}) -# Executes: encode → sample → decode (model already cached!) +result2 = run(handler_stack(workflow_with_prompt("a dog"))) +# Executes: encode -> sample -> decode (model already cached!) -result3 = await runtime.run(workflow, inputs={"prompt": "a dog"}) +result3 = run(handler_stack(workflow_with_prompt("a dog"))) # Executes: nothing (fully cached!) ``` @@ -470,12 +435,12 @@ Since workflows are just Python/Hy functions: | Feature | doeff-flow | ComfyUI | |---------|------------|---------| -| Autocomplete | ✅ | ❌ JSON blob | -| Type checking | ✅ | ❌ String types | -| Go to definition | ✅ | ❌ | -| Find references | ✅ | ❌ | -| Refactoring | ✅ | ❌ | -| Linting | ✅ | ❌ | +| Autocomplete | Yes | No (JSON blob) | +| Type checking | Yes | No (string types) | +| Go to definition | Yes | No | +| Find references | Yes | No | +| Refactoring | Yes | No | +| Linting | Yes | No | ### 11. Version Control @@ -509,7 +474,7 @@ redis.publish("gpu-jobs", serialized) # Worker receives and executes workflow = cloudpickle.loads(message) -result = await runtime.run(workflow) +result = run(handler_stack(workflow)) ``` ComfyUI: Tightly coupled to its server architecture. @@ -518,18 +483,17 @@ ComfyUI: Tightly coupled to its server architecture. | Strength | ComfyUI | doeff-flow | |----------|---------|------------| -| **Testability** | ❌ GPU required | ✅ Mock handlers | -| **Composability** | ❌ Flat nodes | ✅ Nested workflows | -| **Control flow** | ❌ No loops/conditionals | ✅ Full Hy/Python | -| **Debugging** | ❌ Console logs | ✅ Call tree, traces | -| **Replay** | ❌ Not possible | ✅ Record/playback | -| **Multiple runtimes** | ❌ One runtime | ✅ Sync/Async/Sim | -| **Middleware** | ❌ None | ✅ Effect interception | -| **Dependency injection** | ❌ Global config | ✅ Ask effect | -| **Incremental execution** | ⚠️ Basic cache | ✅ Content-addressed | -| **IDE support** | ❌ JSON blobs | ✅ Full support | -| **Version control** | ❌ ID soup | ✅ Named symbols | -| **Distribution** | ❌ Server-coupled | ✅ Serializable | +| **Testability** | No (GPU required) | Yes (mock handlers) | +| **Composability** | No (flat nodes) | Yes (nested workflows) | +| **Control flow** | No (no loops/conditionals) | Yes (full Hy/Python) | +| **Debugging** | No (console logs) | Yes (WithObserve) | +| **Replay** | No (not possible) | Yes (record/playback) | +| **Middleware** | No (none) | Yes (WithObserve) | +| **Dependency injection** | No (global config) | Yes (Ask effect) | +| **Incremental execution** | Partial (basic cache) | Yes (content-addressed) | +| **IDE support** | No (JSON blobs) | Yes (full support) | +| **Version control** | No (ID soup) | Yes (named symbols) | +| **Distribution** | No (server-coupled) | Yes (serializable) | ## Implementation Plan @@ -544,7 +508,7 @@ ComfyUI: Tightly coupled to its server architecture. ### Phase 2: Runtime Integration - [ ] Graph-aware runtime — execute with node-level granularity -- [ ] Event emission — hook into effect execution +- [ ] Event emission — hook into effect execution via `WithObserve` - [ ] Progress reporting — for long-running operations - [ ] Node-level caching — skip unchanged nodes @@ -567,30 +531,30 @@ ComfyUI: Tightly coupled to its server architecture. ``` packages/doeff-flow/ -├── src/doeff_flow/ -│ ├── __init__.py -│ ├── hy/ -│ │ ├── macros.hy # defnode, defeffect, defgraph -│ │ ├── widgets.hy # widget type definitions -│ │ └── prelude.hy # common imports -│ ├── registry.py # node/effect registry -│ ├── schema.py # JSON schema generation -│ ├── runtime.py # graph-aware executor -│ ├── events.py # event emission -│ └── server/ -│ ├── __init__.py -│ ├── api.py # REST endpoints -│ └── ws.py # WebSocket handler -├── frontend/ # or separate package -│ ├── package.json -│ └── src/ -└── examples/ - ├── nodes/ - │ ├── image.hy - │ ├── text.hy - │ └── sampling.hy - └── workflows/ - └── txt2img.hy ++-- src/doeff_flow/ +| +-- __init__.py +| +-- hy/ +| | +-- macros.hy # defnode, defeffect, defgraph +| | +-- widgets.hy # widget type definitions +| | +-- prelude.hy # common imports +| +-- registry.py # node/effect registry +| +-- schema.py # JSON schema generation +| +-- runtime.py # graph-aware executor +| +-- events.py # event emission +| +-- server/ +| +-- __init__.py +| +-- api.py # REST endpoints +| +-- ws.py # WebSocket handler ++-- frontend/ # or separate package +| +-- package.json +| +-- src/ ++-- examples/ + +-- nodes/ + | +-- image.hy + | +-- text.hy + | +-- sampling.hy + +-- workflows/ + +-- txt2img.hy ``` ## What doeff Already Provides @@ -598,14 +562,11 @@ packages/doeff-flow/ | Need | doeff Has | |------|-----------| | Lazy execution | `Program` is lazy | -| Effect interception | `.intercept()` | +| Effect observation | `WithObserve(observer, body)` | | Caching | `@cache` decorator | | Parallel execution | `Gather` effect | -| Graph tracking | `WGraph`, `WStep`, `WNode` | -| Async runtime | `AsyncRuntime` | -| Effect observation | `EffectObservation` | -| Call tree | `EffectCallTree` | -| Visualization | `graph_snapshot.py` → vis.js | +| Async scheduling | `run(scheduled(...))` | +| Handler composition | `handler(raw)(body)` stacking | ## Open Questions @@ -625,15 +586,15 @@ Make every node, edge, and value content-addressed by its hash, like Git/IPFS/Un ``` Current (name-based): - load_model("sdxl") → cache key = ("load_model", "sdxl") + load_model("sdxl") -> cache key = ("load_model", "sdxl") Problem: If load_model code changes, cache is stale Better (content-addressed): Node hash = sha256(node_code + dependencies + inputs) Benefits: - - Code change → hash change → auto-invalidate - - Identical computations across workflows → deduplicated + - Code change -> hash change -> auto-invalidate + - Identical computations across workflows -> deduplicated - Shareable cache across users (like Nix store) - Immutable history (like Git) - Distributed storage (like IPFS) @@ -655,7 +616,7 @@ Instead of "run workflow, skip cached nodes," adopt a **spreadsheet model** wher (def image (derived [model prompt] (generate model prompt))) -;; Change prompt → image automatically recomputes +;; Change prompt -> image automatically recomputes (reset! prompt "a dog") ;; Only encode + sample + decode run ``` @@ -670,21 +631,21 @@ Instead of bidirectional sync between text and UI (which can be lossy), make the ``` Current: Bidirectional sync (lossy) - Hy Code ←→ Visual UI + Hy Code <-> Visual UI Comments lost. Formatting lost. Sync bugs. Future: Projectional editing - ┌─────────────┐ - │ AST │ ← Single source - └─────────────┘ + +--------------+ + | AST | <- Single source + +--------------+ / \ - ↓ ↓ - ┌────────┐ ┌────────┐ - │Text │ │Visual │ ← Views (projections) - │View │ │View │ - └────────┘ └────────┘ + v v + +--------+ +--------+ + |Text | |Visual | <- Views (projections) + |View | |View | + +--------+ +--------+ - Edit in either view → updates AST → other view updates + Edit in either view -> updates AST -> other view updates No sync issues. Perfect round-trip. ``` @@ -694,17 +655,6 @@ Future: Projectional editing doeff uses algebraic effects with one-shot continuations. Multi-shot continuations would enable even more composable handling: -```hy -;; Current: Global handler for effect type -(runtime.run workflow :handlers {Sample: my_handler}) - -;; Future: Scoped, composable handlers -(with-handler [Sample my_handler] - (with-handler [Log silent_handler] ; nested scope! - (run inner-workflow)) - (run outer-workflow)) ; different handler -``` - Multi-shot continuations would enable: - **Multi-shot handlers** — run continuation multiple times (doeff currently supports one-shot only) - **Backtracking** — explore multiple branches from a single effect point @@ -719,7 +669,7 @@ Allow incomplete workflows with "holes" that show what's needed: ```hy (defgraph my-workflow [] (let [model (yield (load-model "sdxl")) - image (yield (generate model ???))] ; ← typed hole + image (yield (generate model ???))] ; <- typed hole image)) ;; System infers: ??? must be type Prompt @@ -734,27 +684,27 @@ Allow incomplete workflows with "holes" that show what's needed: Track where every value came from for debugging and explainability: ```python -result = await runtime.run(workflow) +result = run(handler_stack(workflow)) # Query provenance of any output result.provenance(output_image.pixel[100, 100]) -# → "This pixel came from: -# → decode(latent[50,50]) -# → sample(step=17, noise=0.3) -# → encode('detailed fur texture') -# → Original prompt word 'fur' at position 3" +# -> "This pixel came from: +# -> decode(latent[50,50]) +# -> sample(step=17, noise=0.3) +# -> encode('detailed fur texture') +# -> Original prompt word 'fur' at position 3" ``` ### Implementation Roadmap | Phase | Enhancement | Effort | Impact | |-------|-------------|--------|--------| -| **Phase 1** | Ship current doeff + Hy design | Done | ★★★★★ | -| **Phase 2** | Content-addressed cache layer | Medium | ★★★★☆ | -| **Phase 3** | Incremental computation for preview | High | ★★★★☆ | -| **Phase 4** | Typed holes for partial workflows | Medium | ★★★☆☆ | -| **Research** | Multi-shot continuations | Very High | ★★★☆☆ | -| **Research** | Projectional editing | Very High | ★★★☆☆ | +| **Phase 1** | Ship current doeff + Hy design | Done | High | +| **Phase 2** | Content-addressed cache layer | Medium | High | +| **Phase 3** | Incremental computation for preview | High | High | +| **Phase 4** | Typed holes for partial workflows | Medium | Medium | +| **Research** | Multi-shot continuations | Very High | Medium | +| **Research** | Projectional editing | Very High | Medium | ### Design Principle @@ -770,8 +720,8 @@ The current design is **principled enough** to be extensible toward these ideals - [litegraph.js](https://github.com/jagenjo/litegraph.js) — graph editor library ### doeff Primitives -- `doeff/graph_snapshot.py` — existing vis.js integration -- `doeff/types.py` — `WGraph`/`WStep`/`WNode` DAG primitives +- `doeff/program.py` — handler installer, `WithHandler`, `WithObserve` +- `doeff_vm` — `EffectBase`, `PyVM`, VM IR nodes - `doeff/effects/` — effect system foundation ### Future Directions Research diff --git a/docs/proposals/002-run-result-printing-ownership-plan.md b/docs/proposals/002-run-result-printing-ownership-plan.md index 23761c817..d0703fc33 100644 --- a/docs/proposals/002-run-result-printing-ownership-plan.md +++ b/docs/proposals/002-run-result-printing-ownership-plan.md @@ -1,14 +1,22 @@ --- title: RunResult Printing Ownership Plan -status: in_progress +status: superseded issue: ISSUE-CORE-525 --- # RunResult Printing Ownership Plan +> **Status note (2026-07)**: This plan is largely **superseded**. `RunResult` no longer exists -- +> `run(doexpr)` returns the raw value directly or raises an exception on error. +> The printing-ownership concern was resolved as part of the broader API simplification: +> `run()` enriches exceptions with `__doeff_traceback__` and prints the doeff traceback to stderr, +> but does not wrap the result in a `RunResult` container. The final "Commit follow-up +> implementation" step was never completed as a separate commit because the changes landed +> as part of the `run()` API redesign. + ## Goal -Make `run()` and `async_run()` pure execution APIs by default: they should return a `RunResult` +Make `run()` a pure execution API by default: it should return the result value without printing doeff traces automatically. The CLI becomes the presentation layer responsible for printing human-readable traces in text mode and embedding them in JSON mode. @@ -19,34 +27,35 @@ No backward-compatibility helpers or opt-in shims for the old default behavior a - [x] Commit current traceback formatting / cache cleanup baseline - [x] Document follow-up plan and status tracking - [x] Change `run()` default behavior to not print doeff traces -- [x] Change `async_run()` default behavior to not print doeff traces - [x] Update CLI flow so text mode prints exactly once and JSON mode stays stderr-clean - [x] Update direct runtime tests to assert no default stderr output - [x] Update CLI tests to assert CLI-owned rendering behavior - [x] Re-review smells and validate targeted suites -- [ ] Commit follow-up implementation +- [ ] ~~Commit follow-up implementation~~ (superseded -- changes landed as part of `run()` API redesign) ## Constraints -- `run()` / `async_run()` remain the supported public execution APIs +- `run()` is the supported public execution API (`async_run()` was removed -- use `run(scheduled(...))`) - CLI text mode should print user-facing failure output - CLI JSON mode should keep stderr clean for doeff trace output - Do not preserve the old default-printing behavior via compatibility helpers ## Implementation Outline -1. Flip the runtime defaults in `doeff/rust_vm.py` so `run()` / `async_run()` do not print by default. -2. Keep explicit internal call sites honest (`sync_run`, CLI helpers, discovery paths) so they do not - rely on runtime-owned printing. -3. In CLI text mode, render the attached doeff traceback exactly once when present; otherwise fall back +1. `run(doexpr)` takes a single argument, returns the raw value, and raises on error. + On error, it enriches the exception with `__doeff_traceback__` and prints the doeff + traceback to stderr. +2. CLI text mode renders the doeff traceback exactly once when present; otherwise falls back to captured Python traceback or `Error: ...`. -4. In CLI JSON mode, include the doeff traceback in payload and avoid stderr trace noise. -5. Update tests to lock the contract from both layers: - - runtime APIs: no default stderr trace output +3. CLI JSON mode includes the doeff traceback in payload and avoids stderr trace noise. +4. Tests lock the contract from both layers: + - runtime API: no default stderr trace output (except on error) - CLI text: one printed doeff trace - CLI JSON: structured payload, clean stderr ## Notes - The prior ISSUE-CORE-525 fixes already cleaned up pending-handler rendering, cache note propagation, - and the stale traceback printing hacks. This plan is only for ownership of printing behavior. + and the stale traceback printing hacks. +- `RunResult` no longer exists as a concept. `run()` returns the raw value directly. +- `async_run()` was removed. Use `run(scheduled(...))` for async programs. diff --git a/docs/seedream.md b/docs/seedream.md index f3bd6ef48..c0026cd20 100644 --- a/docs/seedream.md +++ b/docs/seedream.md @@ -26,9 +26,9 @@ program in one of two ways: `seedream_client` key. ```python -import asyncio - -from doeff import async_run, default_async_handlers, do +from doeff import do, run +from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.scheduler import scheduled from doeff_seedream import edit_image__seedream4 @do @@ -39,14 +39,11 @@ def main(): ) result.images[0].save("harbour.png") -run_result = asyncio.run( - async_run( - main(), - handlers=default_async_handlers(), - env={"seedream_api_key": "YOUR_ARK_KEY"}, - ) -) -run_result.value +prog = main() +prog = writer(prog) +prog = state()(prog) +prog = reader(env={"seedream_api_key": "YOUR_ARK_KEY"})(prog) +run(scheduled(prog)) ``` ## Controlling the request payload @@ -85,8 +82,8 @@ atomic shared state keys: - ``seedream_cost_`` – model-specific totals. - ``seedream_api_calls`` – append-only list capturing per-call cost metadata. -Every execution also logs the cost via ``Log`` so that ledger entries surface in -``RunResult.log``. +Every execution also logs the cost via ``Tell`` so that ledger entries surface in +the writer handler's log. ## Request flow diagram @@ -105,7 +102,7 @@ ASCII overview: | v +------------------+ +---------------------+ +------------------+ -| RunResult / Step | <---- | _decode_images(...) | <---- | JSON response | +| Result | <---- | _decode_images(...) | <---- | JSON response | +------------------+ +---------------------+ +------------------+ ``` diff --git a/docs/specs/SPEC-WITH-INTERCEPT.md b/docs/specs/SPEC-WITH-INTERCEPT.md index a9a813080..b68e53682 100644 --- a/docs/specs/SPEC-WITH-INTERCEPT.md +++ b/docs/specs/SPEC-WITH-INTERCEPT.md @@ -17,6 +17,8 @@ The specific motivating use case from this spec — observing effects emitted by handlers above an observer in the stack — is solved by the override pattern because the override handler is installed as the innermost handler for the observed effect type, catching effects before the original handler. +**Current API**: The public Python API replacement is `WithObserve(observer, body)` (exported from `doeff`). See `docs/09-advanced-effects.md` for usage. + **Historical context**: This spec is retained for historical reference. The analysis in "Why Intercept cannot be an Effect" (§Design Rationale) remains valid for the old continuation model. The override pattern avoids the problem entirely by using standard handler semantics (handler + mask) instead of a separate interception mechanism. See: SPEC-008 R17 (R17-F), SPEC-VM-016 R2 (§4.3) diff --git a/docs/specs/SPEC-WITHHANDLER-TYPE-FILTER.md b/docs/specs/SPEC-WITHHANDLER-TYPE-FILTER.md index fdf4acaaf..84329520d 100644 --- a/docs/specs/SPEC-WITHHANDLER-TYPE-FILTER.md +++ b/docs/specs/SPEC-WITHHANDLER-TYPE-FILTER.md @@ -171,14 +171,21 @@ fn should_dispatch_to_handler(effect: &PyAny, handler: &InstalledHandler) -> boo ## Python wrapper changes -### `doeff/rust_vm.py` +### `doeff/program.py` (handler installer) + +The `handler()` function in `doeff/program.py` wraps a raw effect dispatcher as a +`Program -> Program` installer. Type extraction is integrated into the `WithHandler` +construction path. ```python -def WithHandler(handler, expr, return_clause=None): - handler = _coerce_handler(handler, api_name="WithHandler", role="handler") - types = _extract_handler_effect_types(handler) - vm = _vm() - return vm.WithHandler(handler, expr, types=types, return_clause=return_clause) +# doeff/program.py — simplified view +from doeff_vm import WithHandler as WithHandlerType + +def handler(raw_handler): + """Wrap a raw effect dispatcher as a Program -> Program handler.""" + def install(body): + return WithHandlerType(raw_handler, body) + return install ``` ### Type extraction implementation @@ -212,7 +219,9 @@ def _extract_handler_effect_types(handler) -> tuple[type, ...] | None: def _resolve_effect_types(annotation) -> tuple[type, ...] | None: """Resolve an annotation to a tuple of concrete effect types, or None for 'all'.""" - from doeff.types import Effect, EffectBase + from doeff_vm import EffectBase + + Effect = EffectBase # doeff.__init__ aliases Effect = EffectBase # Base classes mean "all effects" if annotation in (Effect, EffectBase, Any): @@ -282,7 +291,7 @@ def _resolve_effect_types(annotation) -> tuple[type, ...] | None: | Phase | Scope | Files | |-------|-------|-------| -| 1 | Python type extraction | `doeff/rust_vm.py` — add `_extract_handler_effect_types` | +| 1 | Python type extraction | `doeff/program.py` or semgrep-guarded utility — add `_extract_handler_effect_types` | | 2 | Rust IR extension | `do_ctrl.rs` — add `types` field to `WithHandler` | | 3 | PyO3 bridge | `pyvm.rs` — accept `types` kwarg on `WithHandler` | | 4 | VM dispatch | `vm.rs` — isinstance pre-check in handler dispatch loop | diff --git a/docs/unified_image_effects.md b/docs/unified_image_effects.md index 5c160b587..5f71de74d 100644 --- a/docs/unified_image_effects.md +++ b/docs/unified_image_effects.md @@ -1,10 +1,15 @@ # Unified Image Effects (`doeff-image`) -`doeff-image` provides provider-agnostic image effects: +`doeff-image` provides provider-agnostic image effects and result types. -- `ImageGenerate` -- `ImageEdit` -- `ImageResult` +## Effects + +- `ImageGenerate` — request image generation from a model +- `ImageEdit` — request editing/transformation of existing images + +## Result Type + +- `ImageResult` — provider-agnostic result dataclass in `doeff_image.types` containing generated/edited images, model name, prompt, and optional metadata (`generation_id`, `cost_usd`, `raw_response`). This is a **data type**, not an effect — it is the return value from handlers that process `ImageGenerate` and `ImageEdit` effects. Provider packages implement protocol handlers and route by model prefix. @@ -52,5 +57,5 @@ result = run( ## Deprecation aliases -- `doeff_seedream.effects.SeedreamGenerate` is a deprecated alias of unified image editing semantics. +- `doeff_seedream.effects.SeedreamGenerate` is a deprecated alias; use `doeff_image.effects.ImageEdit` instead. Instantiation emits a `DeprecationWarning`. - `doeff_gemini.effects.GeminiImageEdit` is a deprecated alias of `doeff_image.effects.ImageEdit`. diff --git a/doeff/cli/run_services.py b/doeff/cli/run_services.py index 6a5cc729b..e5edf5124 100644 --- a/doeff/cli/run_services.py +++ b/doeff/cli/run_services.py @@ -188,7 +188,7 @@ def default_interpreter(program: Any) -> Any: from doeff import run handlers = [ - lazy_ask(), state(), writer(), try_handler, slog_handler(), + lazy_ask(), state(), writer, try_handler, slog_handler, listen_handler, await_handler(), ] wrapped = program diff --git a/packages/doeff-conductor/src/doeff_conductor/handlers/exec_handler.py b/packages/doeff-conductor/src/doeff_conductor/handlers/exec_handler.py index bce6db7f9..b3568c881 100644 --- a/packages/doeff-conductor/src/doeff_conductor/handlers/exec_handler.py +++ b/packages/doeff-conductor/src/doeff_conductor/handlers/exec_handler.py @@ -2,7 +2,6 @@ import os import subprocess -import sys import tempfile from collections.abc import Callable from pathlib import Path @@ -75,12 +74,13 @@ def handle_exec(self, effect: "Exec") -> ExecResult: with log_path.open("w", encoding="utf-8") as log_file: log_file.write(output) - if output: - sys.stdout.write(output) - sys.stdout.flush() - exit_code = process.returncode if timed_out: exit_code = 124 - return ExecResult(exit_code=exit_code, log_path=str(log_path), timed_out=timed_out) + return ExecResult( + exit_code=exit_code, + log_path=str(log_path), + output=output, + timed_out=timed_out, + ) diff --git a/packages/doeff-conductor/src/doeff_conductor/types.py b/packages/doeff-conductor/src/doeff_conductor/types.py index 27ae65475..b58668946 100644 --- a/packages/doeff-conductor/src/doeff_conductor/types.py +++ b/packages/doeff-conductor/src/doeff_conductor/types.py @@ -76,6 +76,7 @@ class ExecResult: exit_code: int log_path: str + output: str = "" timed_out: bool = False @property diff --git a/packages/doeff-core-effects/doeff_core_effects/__init__.py b/packages/doeff-core-effects/doeff_core_effects/__init__.py index 03a677f5b..9d0147abe 100644 --- a/packages/doeff-core-effects/doeff_core_effects/__init__.py +++ b/packages/doeff-core-effects/doeff_core_effects/__init__.py @@ -31,9 +31,11 @@ local_handler, reader, slog_handler, + slog_log, state, try_handler, writer, + writer_log, ) from doeff_core_effects.http_handlers import ( # noqa: F401 http_fixture_handler, diff --git a/packages/doeff-core-effects/doeff_core_effects/handlers.py b/packages/doeff-core-effects/doeff_core_effects/handlers.py index fc056ee8e..2d9552497 100644 --- a/packages/doeff-core-effects/doeff_core_effects/handlers.py +++ b/packages/doeff-core-effects/doeff_core_effects/handlers.py @@ -1,12 +1,18 @@ """ Core handlers — reader, state, writer. -Handler factories return Program -> Program installers. Compose them by calling -the returned handler with the program to wrap: +Stateless handlers are pre-installed Program -> Program functions. +Parameterised handlers are factories that return Program -> Program installers. - prog = state(initial={"count": 0})(body()) +Compose them by calling each handler with the program to wrap: + + prog = writer(state(initial={"count": 0})(body())) prog = reader(env={"key": "value"})(prog) run(prog) + +writer and slog_handler use lazy state init via Get/Put + Some +(same pattern as Hy defhandler's ``lazy`` clause). They require +the ``state`` handler to be installed as an outer handler. """ from doeff import do @@ -69,25 +75,53 @@ def handler(effect, k): return _program_handler(handler) -def writer(): +_WRITER_LOG_KEY = "__doeff_writer_log__" + + +@do +def _writer_handler(effect, k): """Writer handler: collects Tell(message) into a log list. - The log is returned as the handler's result when the body completes. - Access via the handler's return value, or inspect handler_log after run. + Uses lazy state init via Get/Put + Some (same pattern as Hy + defhandler's ``lazy`` clause). Requires the ``state`` handler + to be installed as an outer handler. + + Retrieve the collected log with ``yield writer_log()``. """ - log = [] + if isinstance(effect, WriterTellEffect): + from doeff.result import Some + + cached = yield Get(_WRITER_LOG_KEY) + if isinstance(cached, Some): + log = cached.value + else: + log = [] + yield Put(_WRITER_LOG_KEY, Some(log)) + log.append(effect.msg) + result = yield Resume(k, None) + return result + yield Pass(effect, k) - @do - def handler(effect, k): - if isinstance(effect, WriterTellEffect): - log.append(effect.msg) - result = yield Resume(k, None) - return result - yield Pass(effect, k) - install = _program_handler(handler) - install.log = log # expose for inspection - return install +writer = _program_handler(_writer_handler) +writer.__name__ = "writer" +writer.__qualname__ = "writer" + + +@do +def writer_log(): + """Return a snapshot of the current writer log from state. + + Requires state handler. Returns an empty list if no Tell has + been issued yet. The returned list is a copy — mutations do not + affect the handler's internal log. + """ + from doeff.result import Some + + cached = yield Get(_WRITER_LOG_KEY) + if isinstance(cached, Some): + return list(cached.value) + return [] @do @@ -132,26 +166,54 @@ def attempt(): try_handler.__qualname__ = "try_handler" -def slog_handler(): +_SLOG_LOG_KEY = "__doeff_slog_log__" + + +@do +def _slog_handler(effect, k): """Structured log handler: collects Slog messages. - Returns a handler with a .log attribute containing collected entries. + Uses lazy state init via Get/Put + Some. Requires the ``state`` + handler to be installed as an outer handler. + + Retrieve the collected log with ``yield slog_log()``. Each entry is a dict with 'msg' and all kwargs. """ - log = [] + if isinstance(effect, Slog): + from doeff.result import Some + + cached = yield Get(_SLOG_LOG_KEY) + if isinstance(cached, Some): + log = cached.value + else: + log = [] + yield Put(_SLOG_LOG_KEY, Some(log)) + entry = {"msg": effect.msg, **effect.kwargs} + log.append(entry) + result = yield Resume(k, None) + return result + yield Pass(effect, k) - @do - def handler(effect, k): - if isinstance(effect, Slog): - entry = {"msg": effect.msg, **effect.kwargs} - log.append(entry) - result = yield Resume(k, None) - return result - yield Pass(effect, k) - install = _program_handler(handler) - install.log = log - return install +slog_handler = _program_handler(_slog_handler) +slog_handler.__name__ = "slog_handler" +slog_handler.__qualname__ = "slog_handler" + + +@do +def slog_log(): + """Return a snapshot of the current structured log from state. + + Requires state handler. Returns an empty list if no Slog has + been issued yet. The returned list is a copy — mutations do not + affect the handler's internal log. + """ + from doeff.result import Some + + cached = yield Get(_SLOG_LOG_KEY) + if isinstance(cached, Some): + return list(cached.value) + return [] @do diff --git a/packages/doeff-docker/src/doeff_docker/compose.hy b/packages/doeff-docker/src/doeff_docker/compose.hy index 5f47a5012..c0214c379 100644 --- a/packages/doeff-docker/src/doeff_docker/compose.hy +++ b/packages/doeff-docker/src/doeff_docker/compose.hy @@ -3,11 +3,11 @@ (defmacro with-handlers [handlers body] "Compose multiple handlers around a body program. - (with-handlers [(writer) (slog-handler) resolve-handler] + (with-handlers [writer slog-handler resolve-handler] (my-program)) Expands to: - ((writer) ((slog-handler) (resolve-handler (my-program)))) + (writer (slog-handler (resolve-handler (my-program)))) Handlers are applied inner-first: last in list is innermost." (setv result body) diff --git a/packages/doeff-docker/src/doeff_docker/effects.hy b/packages/doeff-docker/src/doeff_docker/effects.hy index 16d7bc51d..352ad8e1e 100644 --- a/packages/doeff-docker/src/doeff_docker/effects.hy +++ b/packages/doeff-docker/src/doeff_docker/effects.hy @@ -71,3 +71,22 @@ "Push a Docker image to a registry." #^ str local-tag #^ str remote-tag) + + +;; =========================================================================== +;; Shell substrate — boundary effect for subprocess execution +;; =========================================================================== + +(defclass [(dataclass :frozen True :kw-only True)] ShellRunResult [] + "Structured result from a ShellRun effect." + #^ int returncode + #^ bytes stdout + #^ bytes stderr) + +(defclass [(dataclass :frozen True :kw-only True)] ShellRun [EffectBase] + "Run a subprocess as a list of string args. A production shell-run-handler + calls subprocess.run — that IS the boundary. Handler clauses yield this + effect instead of calling subprocess directly." + #^ tuple args + #^ (| bytes None) stdin-data + (setv stdin-data None)) diff --git a/packages/doeff-docker/src/doeff_docker/handlers/docker.hy b/packages/doeff-docker/src/doeff_docker/handlers/docker.hy index edede70dc..25c72bb21 100644 --- a/packages/doeff-docker/src/doeff_docker/handlers/docker.hy +++ b/packages/doeff-docker/src/doeff_docker/handlers/docker.hy @@ -1,30 +1,25 @@ ;;; Docker operation handlers ;;; Handles DockerBuild, ImagePush effects via shell commands. ;;; DockerRun is application-specific (depends on runner module) — not included here. +;;; +;;; Handler clauses yield ShellRun effects instead of calling subprocess +;;; directly — the production shell-run-handler (handlers/shell.hy) is the +;;; IO boundary that resolves them. (require doeff_hy.macros [<- defhandler]) (import doeff [do :as _doeff-do]) (import doeff_core_effects [slog]) -(import subprocess) (import shlex) -(import pathlib [Path]) -(import doeff_docker.effects [DockerBuild ImagePush]) +(import doeff_docker.effects [DockerBuild ImagePush ShellRun]) -;; Plain callable: shared synchronous subprocess boundary used by handler clauses. -(defn run-cmd [args * [host "localhost"] [stdin-data None]] - "Run a command as list of args, optionally on a remote host via SSH. - Returns CompletedProcess." - (setv full-args - (if (= host "localhost") - args - ["ssh" host (.join " " (lfor a args (shlex.quote (str a))))])) - (setv proc (subprocess.run full-args :capture-output True :input stdin-data)) - (when (!= proc.returncode 0) - (raise (RuntimeError f"Command failed (rc={proc.returncode}):\n{full-args}\nstderr: {(.decode proc.stderr)}"))) - proc) +(defn _build-shell-args [args * [host "localhost"]] + "Build full command args, wrapping in SSH if host is not localhost." + (if (= host "localhost") + (list args) + ["ssh" host (.join " " (lfor a args (shlex.quote (str a))))])) (defhandler docker-build-handler @@ -32,8 +27,12 @@ (DockerBuild [dockerfile tag context-path host] (<- (slog :msg f"docker build: {tag} on {host}")) (setv args ["docker" "build" "-t" tag "-f" "-" (str context-path)]) - (run-cmd args :host host - :stdin-data (.encode dockerfile "utf-8")) + (setv full-args (_build-shell-args args :host host)) + (<- result (ShellRun :args (tuple full-args) + :stdin-data (.encode dockerfile "utf-8"))) + (when (!= result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={result.returncode}):\n{full-args}\nstderr: {(.decode result.stderr)}"))) (resume tag))) @@ -41,6 +40,16 @@ "Handle ImagePush: tag and push image to registry." (ImagePush [local-tag remote-tag] (<- (slog :msg f"docker push: {local-tag} -> {remote-tag}")) - (run-cmd ["docker" "tag" local-tag remote-tag]) - (run-cmd ["docker" "push" remote-tag]) + ;; tag + (setv tag-args ["docker" "tag" local-tag remote-tag]) + (<- tag-result (ShellRun :args (tuple tag-args))) + (when (!= tag-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={tag-result.returncode}):\n{tag-args}\nstderr: {(.decode tag-result.stderr)}"))) + ;; push + (setv push-args ["docker" "push" remote-tag]) + (<- push-result (ShellRun :args (tuple push-args))) + (when (!= push-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={push-result.returncode}):\n{push-args}\nstderr: {(.decode push-result.stderr)}"))) (resume remote-tag))) diff --git a/packages/doeff-docker/src/doeff_docker/handlers/shell.hy b/packages/doeff-docker/src/doeff_docker/handlers/shell.hy new file mode 100644 index 000000000..1db4c0658 --- /dev/null +++ b/packages/doeff-docker/src/doeff_docker/handlers/shell.hy @@ -0,0 +1,25 @@ +;;; Shell subprocess handler — the IO boundary for ShellRun effects. +;;; +;;; This is the production handler that actually calls subprocess.run. +;;; Handler clauses in docker.hy, rsync.hy, file.hy yield ShellRun effects +;;; instead of calling subprocess directly; this handler sits at the outer +;;; boundary of the handler stack and resolves those effects. + +(require doeff_hy.macros [defhandler]) +(import doeff [do :as _doeff-do]) + +(import subprocess) + +(import doeff_docker.effects [ShellRun ShellRunResult]) + + +(defhandler shell-run-handler + "Handle ShellRun: execute subprocess and return ShellRunResult. + This is the IO boundary — the only place subprocess.run is called." + (ShellRun [args stdin-data] + (setv proc (subprocess.run (list args) + :capture-output True + :input stdin-data)) + (resume (ShellRunResult :returncode proc.returncode + :stdout proc.stdout + :stderr proc.stderr)))) diff --git a/packages/doeff-docker/tests/test_effects.py b/packages/doeff-docker/tests/test_effects.py index dac6df676..34257e304 100644 --- a/packages/doeff-docker/tests/test_effects.py +++ b/packages/doeff-docker/tests/test_effects.py @@ -16,7 +16,7 @@ def _run_with_handlers(program): return run( - writer()(slog_handler()(reader(env={})(program))) + writer(slog_handler(reader(env={})(program))) ) diff --git a/packages/doeff-llm/README.md b/packages/doeff-llm/README.md index 874a0f89f..d7e0ba035 100644 --- a/packages/doeff-llm/README.md +++ b/packages/doeff-llm/README.md @@ -10,7 +10,7 @@ that route these effects by model name. - `LLMChat` - `LLMStreamingChat` -- `LLMStructuredOutput` +- `LLMStructuredQuery` - `LLMEmbedding` ## Quick Example @@ -40,6 +40,5 @@ result = run( gemini_production_handler, WithHandler(openai_production_handler, workflow()), ), - env={"openai_api_key": "...", "gemini_api_key": "..."}, ) ``` diff --git a/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/docker.hy b/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/docker.hy index 0566f365f..a08bec07d 100644 --- a/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/docker.hy +++ b/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/docker.hy @@ -1,6 +1,10 @@ ;;; ML-nexus Docker run handler. ;;; DockerBuild and ImagePush handlers are in doeff-docker. ;;; This module provides DockerRun handler with file-based cloudpickle exchange. +;;; +;;; Handler clauses yield ShellRun effects instead of calling subprocess +;;; directly — the production shell-run-handler resolves them at the IO +;;; boundary. (require doeff_hy.macros [<- defhandler]) (import doeff [do :as _doeff-do]) @@ -8,8 +12,13 @@ (import uuid) -(import doeff_docker.effects [DockerRun]) -(import doeff_docker.handlers.docker [run-cmd]) +(import doeff_docker.effects [DockerRun ShellRun]) +(import doeff_docker.handlers.docker [_build-shell-args]) + + +(defn _shell-run-checked [full-args * [stdin-data None]] + "Build a ShellRun effect from args. Caller must yield and check result." + (ShellRun :args (tuple full-args) :stdin-data stdin-data)) (defhandler docker-run-handler @@ -29,9 +38,18 @@ (setv container-exchange "/tmp/doeff-exchange") ;; Create tmp dir and write pickled program - (run-cmd ["mkdir" "-p" tmp-dir] :host host) + (setv mkdir-args (_build-shell-args ["mkdir" "-p" tmp-dir] :host host)) + (<- mkdir-result (ShellRun :args (tuple mkdir-args))) + (when (!= mkdir-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={mkdir-result.returncode}):\n{mkdir-args}\nstderr: {(.decode mkdir-result.stderr)}"))) + (setv pickled (.dumps serializer program)) - (run-cmd ["tee" f"{tmp-dir}/program.pkl"] :host host :stdin-data pickled) + (setv tee-args (_build-shell-args ["tee" f"{tmp-dir}/program.pkl"] :host host)) + (<- tee-result (ShellRun :args (tuple tee-args) :stdin-data pickled)) + (when (!= tee-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={tee-result.returncode}):\n{tee-args}\nstderr: {(.decode tee-result.stderr)}"))) ;; Docker run args (setv parts ["docker" "run" "--rm"]) @@ -52,13 +70,23 @@ "--interpreter" "doeff_ml_nexus.runner.runner_interpreter"]) ;; Execute - (run-cmd parts :host host) + (setv run-args (_build-shell-args parts :host host)) + (<- run-result (ShellRun :args (tuple run-args))) + (when (!= run-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={run-result.returncode}):\n{run-args}\nstderr: {(.decode run-result.stderr)}"))) ;; Read result - (setv proc (run-cmd ["cat" f"{tmp-dir}/result.pkl"] :host host)) - (setv result (.loads serializer proc.stdout)) + (setv cat-args (_build-shell-args ["cat" f"{tmp-dir}/result.pkl"] :host host)) + (<- cat-result (ShellRun :args (tuple cat-args))) + (when (!= cat-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={cat-result.returncode}):\n{cat-args}\nstderr: {(.decode cat-result.stderr)}"))) + (setv result (.loads serializer cat-result.stdout)) ;; Cleanup - (run-cmd ["rm" "-rf" tmp-dir] :host host) + (setv rm-args (_build-shell-args ["rm" "-rf" tmp-dir] :host host)) + (<- rm-result (ShellRun :args (tuple rm-args))) + ;; Cleanup failure is non-fatal — do not raise (resume result))) diff --git a/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/file.hy b/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/file.hy index 92766a0a1..8d6252446 100644 --- a/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/file.hy +++ b/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/file.hy @@ -1,21 +1,27 @@ ;;; File operation handlers -;;; Handles WriteFile effect. +;;; Handles WriteFile effect via ShellRun effects. +;;; +;;; Handler clauses yield ShellRun instead of calling subprocess/Path IO +;;; directly. The production shell-run-handler resolves them at the IO +;;; boundary. -(require doeff_hy.macros [defhandler]) +(require doeff_hy.macros [<- defhandler]) (import doeff [do :as _doeff-do]) -(import subprocess) -(import pathlib [Path]) - +(import doeff_docker.effects [ShellRun]) (import doeff_ml_nexus.effects [WriteFile]) (defhandler write-file-handler "Handle WriteFile: write content to a file on a host." (WriteFile [host path content] - (if (= host "localhost") - (.write-text (Path path) content) - (subprocess.run ["ssh" host f"cat > {path}"] - :input (.encode content) :check True - :capture-output True)) + (setv encoded (.encode content)) + (setv args + (if (= host "localhost") + #("tee" path) + #("ssh" host f"cat > {path}"))) + (<- result (ShellRun :args args :stdin-data encoded)) + (when (!= result.returncode 0) + (raise (RuntimeError + f"WriteFile failed (rc={result.returncode}):\n{(list args)}\nstderr: {(.decode result.stderr)}"))) (resume path))) diff --git a/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/rsync.hy b/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/rsync.hy index 761a16000..8beb49113 100644 --- a/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/rsync.hy +++ b/packages/doeff-ml-nexus/src/doeff_ml_nexus/handlers/rsync.hy @@ -1,13 +1,16 @@ ;;; Rsync handler -;;; Handles RsyncTo effect via rsync shell command. +;;; Handles RsyncTo effect via ShellRun effects. +;;; +;;; Handler clauses yield ShellRun instead of calling subprocess directly. +;;; The production shell-run-handler resolves them at the IO boundary. (require doeff_hy.macros [defk <- defhandler]) (import doeff [do :as _doeff-do]) (import doeff_core_effects [slog]) -(import subprocess) (import pathlib [Path]) +(import doeff_docker.effects [ShellRun]) (import doeff_ml_nexus.effects [RsyncTo]) @@ -40,10 +43,16 @@ (<- (slog :msg f"rsync: {src} -> {host}:{dst-path}")) ;; Ensure destination directory exists (when (!= host "localhost") - (subprocess.run ["ssh" host f"mkdir -p {dst-path}"] - :check True :capture-output True)) + (setv mkdir-args ["ssh" host f"mkdir -p {dst-path}"]) + (<- mkdir-result (ShellRun :args (tuple mkdir-args))) + (when (!= mkdir-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={mkdir-result.returncode}):\n{mkdir-args}\nstderr: {(.decode mkdir-result.stderr)}")))) (<- args (_rsync-args src host dst-path :excludes excludes :includes includes)) - (subprocess.run args :check True :capture-output True) + (<- rsync-result (ShellRun :args (tuple args))) + (when (!= rsync-result.returncode 0) + (raise (RuntimeError + f"Command failed (rc={rsync-result.returncode}):\n{args}\nstderr: {(.decode rsync-result.stderr)}"))) (resume dst-path))) diff --git a/packages/doeff-ml-nexus/src/doeff_ml_nexus/runner.hy b/packages/doeff-ml-nexus/src/doeff_ml_nexus/runner.hy index c53360daf..500ef2d8b 100644 --- a/packages/doeff-ml-nexus/src/doeff_ml_nexus/runner.hy +++ b/packages/doeff-ml-nexus/src/doeff_ml_nexus/runner.hy @@ -48,5 +48,5 @@ "Minimal interpreter for the runner program." (setv resolved-env (run (resolve-runner-env env))) (run (scheduled - (with-handlers [(reader :env resolved-env) (slog-handler) (writer)] + (with-handlers [(reader :env resolved-env) slog-handler writer] program)))) diff --git a/packages/doeff-ml-nexus/src/doeff_ml_nexus/stack.hy b/packages/doeff-ml-nexus/src/doeff_ml_nexus/stack.hy index aae0acf4f..7ce97d86a 100644 --- a/packages/doeff-ml-nexus/src/doeff_ml_nexus/stack.hy +++ b/packages/doeff-ml-nexus/src/doeff_ml_nexus/stack.hy @@ -13,6 +13,7 @@ (import doeff_ml_nexus.handlers.rsync [rsync-handler]) (import doeff_ml_nexus.handlers.file [write-file-handler]) (import doeff_docker.handlers.docker [docker-build-handler image-push-handler]) +(import doeff_docker.handlers.shell [shell-run-handler]) (defn ml-nexus-interpreter [program * [env None]] ; doeff: interpreter @@ -28,12 +29,13 @@ (run (scheduled (with-handlers [(reader :env resolved-env) - (slog-handler) - (writer) + slog-handler + writer resolve-handler write-file-handler rsync-handler image-push-handler docker-build-handler - docker-run-handler] + docker-run-handler + shell-run-handler] program)))) diff --git a/packages/doeff-ml-nexus/tests/test_docker.py b/packages/doeff-ml-nexus/tests/test_docker.py index 3f57e1ccd..2b8188569 100644 --- a/packages/doeff-ml-nexus/tests/test_docker.py +++ b/packages/doeff-ml-nexus/tests/test_docker.py @@ -11,7 +11,7 @@ def _run_with_handlers(program): return run( - writer()(slog_handler()(reader(env={})(program))) + writer(slog_handler(reader(env={})(program))) ) diff --git a/packages/doeff-openai/tests/_runner.py b/packages/doeff-openai/tests/_runner.py index 33db57beb..4bae3ecb0 100644 --- a/packages/doeff-openai/tests/_runner.py +++ b/packages/doeff-openai/tests/_runner.py @@ -30,6 +30,7 @@ state, try_handler, writer, + writer_log, ) from doeff_core_effects.scheduler import scheduled from doeff_openai.handlers import calculate_cost_handler @@ -56,23 +57,16 @@ def is_err(self) -> bool: def _build_chain(program: Any, env: dict | None): """Build the legacy default handler chain. - Returns ``(wrapped_program, writer_ref, slog_ref)`` — the two refs let - the caller read the accumulated ``WriterTellEffect`` / ``Slog`` log - after ``run`` returns. Composition order mirrors the working pattern - in ``doeff-traverse/tests/test_traverse_deep_recursion.py`` — - ``scheduled`` is the outermost wrapper, and every in-chain - ``try_handler`` / ``listen_handler`` / ``state`` / etc. sits inside - it so scheduler-owned effects resolve correctly regardless of which - Try/Listen scope they're nested in. - - ``slog_handler`` is installed but positioned *outside* ``writer`` — - writer captures the ``Tell`` stream first (so it shows up in - ``RunResult.log``) and then Pass-es through. ``slog_handler`` is - retained for tests that explicitly emit structured ``slog`` calls - with kwargs. + Returns a single wrapped program. The writer log is captured inside + the program via ``writer_log()`` — no side-channel ``.log`` access. + + Composition order mirrors ``doeff-traverse/tests`` — ``scheduled`` + is outermost so scheduler effects resolve regardless of scope. + + ``slog_handler`` is positioned outside ``writer`` — writer captures + the ``Tell`` stream and passes through; ``slog_handler`` handles + structured ``slog`` calls. """ - writer_h = writer() - slog_h = slog_handler() wrapped = program wrapped = calculate_cost_handler(wrapped) wrapped = await_handler()(wrapped) @@ -80,10 +74,10 @@ def _build_chain(program: Any, env: dict | None): wrapped = local_handler(wrapped) wrapped = state()(wrapped) wrapped = try_handler(wrapped) - wrapped = writer_h(wrapped) - wrapped = slog_h(wrapped) + wrapped = writer(wrapped) + wrapped = slog_handler(wrapped) wrapped = lazy_ask(env=env or {})(wrapped) - return scheduled(wrapped), writer_h, slog_h + return scheduled(wrapped) async def run_program(program: Any, env: dict | None = None) -> RunResult: @@ -96,16 +90,18 @@ async def run_program(program: Any, env: dict | None = None) -> RunResult: @do def _wrap(): - return (yield Try(program)) + outcome = yield Try(program) + log = yield writer_log() + return (outcome, log) - chain, writer_h, _slog_h = _build_chain(_wrap(), env) - outcome = run(chain) + chain = _build_chain(_wrap(), env) + result = run(chain) + outcome, log = result - log = list(writer_h.log) if isinstance(outcome, Ok): - return RunResult(value=outcome.value, log=log) + return RunResult(value=outcome.value, log=list(log)) if isinstance(outcome, Err): - return RunResult(error=outcome.error, log=log) + return RunResult(error=outcome.error, log=list(log)) raise RuntimeError( f"unexpected Try outcome: {type(outcome).__name__} — expected Ok/Err" ) diff --git a/packages/doeff-time/tests/conftest.py b/packages/doeff-time/tests/conftest.py index c46d5fbdc..7fe6daf33 100644 --- a/packages/doeff-time/tests/conftest.py +++ b/packages/doeff-time/tests/conftest.py @@ -71,8 +71,8 @@ def run_with_handlers(program, *, env=None): # so it must be inside the scheduler. wrapped = await_handler()(wrapped) wrapped = scheduled(wrapped) - wrapped = slog_handler()(wrapped) + wrapped = slog_handler(wrapped) wrapped = try_handler(wrapped) wrapped = listen_handler(wrapped) - wrapped = writer()(wrapped) + wrapped = writer(wrapped) return run(wrapped) diff --git a/packages/doeff-traverse/tests/test_memory_leak_multi_day.py b/packages/doeff-traverse/tests/test_memory_leak_multi_day.py index 71a43aa88..c556d460a 100644 --- a/packages/doeff-traverse/tests/test_memory_leak_multi_day.py +++ b/packages/doeff-traverse/tests/test_memory_leak_multi_day.py @@ -32,7 +32,6 @@ from doeff_time.handlers import sim_time_handler from doeff_traverse.effects import Inspect, Traverse from doeff_traverse.handlers import fail_handler, parallel -from doeff_vm import WithHandler from doeff import EffectBase, Pass, Program, Resume, do, run, slog from doeff import handler as _program_handler @@ -108,9 +107,15 @@ def _multi_day_program(n_days: int, n_items_per_day: int) -> Program[int]: def _compose(program, *handlers): + """Compose handlers onto program. + + Each handler is a Program -> Program function (the result of + ``doeff.handler(raw_handler)`` or a pre-installed handler like + ``writer``, ``slog_handler``). + """ wrapped = program for h in reversed(handlers): - wrapped = WithHandler(h, wrapped) + wrapped = h(wrapped) return wrapped @@ -119,7 +124,7 @@ def _run_multi_day(n_days, n_items_per_day, concurrency=20): wrapped = _compose( _multi_day_program(n_days, n_items_per_day), lazy_ask(env={"config": "test"}), - writer(), + writer, try_handler, state(), local_handler, @@ -129,7 +134,7 @@ def _run_multi_day(n_days, n_items_per_day, concurrency=20): _make_mock_handler(), parallel(concurrency=concurrency), fail_handler, - slog_handler(), + slog_handler, ) return run(scheduled(wrapped)) diff --git a/packages/doeff-traverse/tests/test_traverse_deep_recursion.py b/packages/doeff-traverse/tests/test_traverse_deep_recursion.py index 3e363ce89..c16d8a74b 100644 --- a/packages/doeff-traverse/tests/test_traverse_deep_recursion.py +++ b/packages/doeff-traverse/tests/test_traverse_deep_recursion.py @@ -30,7 +30,6 @@ from doeff_time import GetTime, WaitUntil, sim_time_handler from doeff_traverse.effects import Inspect, Traverse from doeff_traverse.handlers import fail_handler, parallel -from doeff_vm import WithHandler from doeff import EffectBase, Pass, Program, Resume, do, run, slog from doeff import handler as _program_handler @@ -64,9 +63,13 @@ def process_item_multi_effect(i: int) -> Program[int]: def _compose(program, *handlers): + """Compose handlers onto program. + + Each handler is a Program -> Program function. + """ wrapped = program for handler in reversed(handlers): - wrapped = WithHandler(handler, wrapped) + wrapped = handler(wrapped) return wrapped @@ -74,7 +77,7 @@ def _run_parallel(program, concurrency=40): wrapped = _compose( program, lazy_ask(env={"multiplier": 3}), - writer(), + writer, try_handler, state(), local_handler, @@ -82,7 +85,7 @@ def _run_parallel(program, concurrency=40): await_handler(), parallel(concurrency=concurrency), fail_handler, - slog_handler(), + slog_handler, ) return run(scheduled(wrapped)) @@ -204,7 +207,7 @@ def _run_parallel_with_sim_time(program, concurrency=40): wrapped = _compose( program, lazy_ask(env={"multiplier": 3}), - writer(), + writer, try_handler, state(), local_handler, @@ -213,7 +216,7 @@ def _run_parallel_with_sim_time(program, concurrency=40): sim_time_handler(start_time=SIM_EPOCH), parallel(concurrency=concurrency), fail_handler, - slog_handler(), + slog_handler, ) return run(scheduled(wrapped)) @@ -326,7 +329,7 @@ def _run_deep_stack(program, n_extra_handlers=15, concurrency=40): wrapped = _compose( program, lazy_ask(env={"multiplier": 3}), - writer(), + writer, try_handler, state(), local_handler, @@ -336,7 +339,7 @@ def _run_deep_stack(program, n_extra_handlers=15, concurrency=40): *extra, parallel(concurrency=concurrency), fail_handler, - slog_handler(), + slog_handler, ) return run(scheduled(wrapped)) @@ -479,7 +482,7 @@ def _run_deep_catching_stack(program, concurrency=40): wrapped = _compose( program, lazy_ask(env={"multiplier": 3}), - writer(), + writer, try_handler, state(), local_handler, @@ -491,7 +494,7 @@ def _run_deep_catching_stack(program, concurrency=40): *extra, parallel(concurrency=concurrency), fail_handler, - slog_handler(), + slog_handler, ) return run(scheduled(wrapped)) diff --git a/packages/doeff-vm/tests/test_pyvm.py b/packages/doeff-vm/tests/test_pyvm.py index b441c1fd2..e7a5054a7 100644 --- a/packages/doeff-vm/tests/test_pyvm.py +++ b/packages/doeff-vm/tests/test_pyvm.py @@ -2,7 +2,7 @@ import doeff_vm import pytest -from doeff_core_effects.handlers import reader, state, writer +from doeff_core_effects.handlers import reader, state, writer, writer_log from doeff_core_effects.scheduler import Gather, Spawn, scheduled from doeff import Ask, Get, Put, Tell, do, run @@ -200,14 +200,12 @@ def reader_body(): def writer_body(): yield Tell("starting") yield Tell("done") - return "ok" - - writer_handler = writer() + log = yield writer_log() + return ("ok", log) assert run(state(initial={"counter": 1})(state_body())) == 2 assert run(reader(env={"name": "Ada"})(reader_body())) == "Ada" - assert run(writer_handler(writer_body())) == "ok" - assert writer_handler.log == ["starting", "done"] + assert run(state()(writer(writer_body()))) == ("ok", ["starting", "done"]) def test_scheduled_spawn_gather_runs_via_doeff_facade() -> None: diff --git a/specs/vm/SPEC-008-rust-vm.md b/specs/vm/SPEC-008-rust-vm.md index cef9822f0..cad48d747 100644 --- a/specs/vm/SPEC-008-rust-vm.md +++ b/specs/vm/SPEC-008-rust-vm.md @@ -256,7 +256,7 @@ def my_program(): program = my_program() -program = writer()(program) +program = writer(program) program = state(initial={"x": 0})(program) program = reader(env={"greeting": "hello"})(program) result = run(program) @@ -1763,7 +1763,7 @@ def user_program(): program = user_program() -program = writer()(program) +program = writer(program) program = state(initial={"x": 0})(program) program = reader(env={"prefix": "count"})(program) diff --git a/tests/_run_helpers.py b/tests/_run_helpers.py index ba5116dae..78a2b745f 100644 --- a/tests/_run_helpers.py +++ b/tests/_run_helpers.py @@ -42,9 +42,9 @@ def default_handlers(env: Any = None, store: Any = None) -> list[Any]: return [ reader(env=env), state(initial=initial), - writer(), + writer, try_handler, - slog_handler(), + slog_handler, local_handler, listen_handler, await_handler(), diff --git a/tests/cli/test_cli_main.py b/tests/cli/test_cli_main.py index fb9719a56..6d0a979f8 100644 --- a/tests/cli/test_cli_main.py +++ b/tests/cli/test_cli_main.py @@ -78,7 +78,7 @@ def _env_interpreter(program, env=None): env_dict = result.value if hasattr(result, "value") else result handlers = [ - reader(env_dict), state(), writer(), try_handler, slog_handler(), + reader(env_dict), state(), writer, try_handler, slog_handler, local_handler, listen_handler, await_handler(), ] wrapped = program diff --git a/tests/conftest.py b/tests/conftest.py index 0fbfe17b2..f149b676a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ def default_handlers(env=None): Includes scheduler + lazy_ask for full effect support. """ return [ - reader(env=env), state(), writer(), try_handler, slog_handler(), + reader(env=env), state(), writer, try_handler, slog_handler, local_handler, listen_handler, await_handler(), lazy_ask(), ] diff --git a/tests/effects/http_request_deftest_cases.hy b/tests/effects/http_request_deftest_cases.hy index 7c2c6d0a5..10b883a46 100644 --- a/tests/effects/http_request_deftest_cases.hy +++ b/tests/effects/http_request_deftest_cases.hy @@ -4,13 +4,13 @@ (import pathlib [Path]) (import pytest) (import doeff_core_effects [HttpError HttpRequest HttpResponse]) -(import doeff_core_effects.handlers [await-handler slog-handler]) +(import doeff_core_effects.handlers [await-handler slog-handler slog-log state]) (import doeff_core_effects.http_handlers [http-production-handler http-fixture-handler]) (import doeff_hy.http [http-get http-post http-put http-delete http-head]) +(import doeff_core_effects.handlers [slog-log]) (import tests.effects.http_request_support [FakeAsyncClient handler-name - handler-log is-doeff-handler make-response noop-sleep @@ -64,15 +64,15 @@ (setv client (FakeAsyncClient [(make-response 200 {"X-Test" "yes"} b"ok" "ok" "https://example.test/final" 0.2)])) - (setv logs (slog-handler)) - (<- response - (logs ((await-handler) + (<- #(response slog-entries) + ((state) (slog-handler ((await-handler) ((http-production-handler :client-factory (fn [] client) :sleep noop-sleep) (do! (<- resp (HttpRequest "GET" "https://example.test/start" :params {"a" "1"})) - resp))))) + (<- log (slog-log)) + #(resp log))))))) (assert (= (. response status) 200)) (assert (= (. response headers) {"X-Test" "yes"})) (assert (= (. response content) b"ok")) @@ -86,7 +86,7 @@ "content" None "timeout" 30.0 "follow_redirects" True}])) - (assert (= (handler-log logs) + (assert (= slog-entries [{"msg" "http_request" "method" "GET" "url" "https://example.test/start" @@ -102,14 +102,14 @@ [(make-response 201 {"Content-Type" "application/json"} b"{}" "{}" "https://example.test/api" 0.1)])) (<- response - ((slog-handler) ((await-handler) + ((state) (slog-handler ((await-handler) ((http-production-handler :client-factory (fn [] client) :sleep noop-sleep) (do! (<- resp (HttpRequest "POST" "https://example.test/api" :headers {"X-Trace" "abc"} :body {"b" 2 "a" 1})) - resp))))) + resp)))))) (assert (= (. response status) 201)) (assert (= (get (get client.calls 0) "headers") {"X-Trace" "abc" "Content-Type" "application/json"})) @@ -122,14 +122,14 @@ [(make-response 302 {"Location" "/next"} b"" "" "https://example.test/start" 0.1)])) (<- response - ((slog-handler) ((await-handler) + ((state) (slog-handler ((await-handler) ((http-production-handler :client-factory (fn [] client) :sleep noop-sleep) (do! (<- resp (HttpRequest "HEAD" "https://example.test/start" :timeout-seconds 1.25 :follow-redirects False)) - resp))))) + resp)))))) (assert (= (. response status) 302)) (assert (= (get (get client.calls 0) "timeout") 1.25)) (assert (is (get (get client.calls 0) "follow_redirects") False)) @@ -143,13 +143,13 @@ (make-response 200 {} b"ok" "ok" "https://example.test/api" 0.1)])) (setv sleeps []) (<- response - ((slog-handler) ((await-handler) + ((state) (slog-handler ((await-handler) ((http-production-handler :client-factory (fn [] client) :sleep (record-sleep sleeps)) (do! (<- resp (HttpRequest "GET" "https://example.test/api" :max-retries 2)) - resp))))) + resp)))))) (assert (= (. response status) 200)) (assert (= (len client.calls) 3)) (assert (= sleeps [0.25 0.5])) @@ -162,14 +162,14 @@ (make-response 200 {} b"ok" "ok" "https://example.test/api" 0.1)])) (setv sleeps []) (<- response - ((slog-handler) ((await-handler) + ((state) (slog-handler ((await-handler) ((http-production-handler :client-factory (fn [] client) :sleep (record-sleep sleeps)) (do! (<- resp (HttpRequest "GET" "https://example.test/api" :timeout-seconds 0.01 :max-retries 1)) - resp))))) + resp)))))) (assert (= (. response status) 200)) (assert (= (len client.calls) 2)) (assert (= (lfor call client.calls (get call "timeout")) [0.01 0.01])) @@ -183,13 +183,13 @@ [(make-response 200 {"X-Fixture" "yes"} b"fixture" "fixture" "https://example.test/resource" 0.3)])) (<- recorded - ((slog-handler) ((await-handler) + ((state) (slog-handler ((await-handler) ((http-fixture-handler fixture-path :mode "record" :client-factory (fn [] client) :sleep noop-sleep) (do! (<- resp (HttpRequest "GET" "https://example.test/resource")) - resp))))) + resp)))))) (<- replayed ((http-fixture-handler fixture-path :mode "replay") (do! diff --git a/tests/effects/http_request_support.hy b/tests/effects/http_request_support.hy index a2aca0ca4..4972e4bd8 100644 --- a/tests/effects/http_request_support.hy +++ b/tests/effects/http_request_support.hy @@ -82,7 +82,8 @@ (defn handler-log [handler] - handler.log) + "DEPRECATED: handler.log is removed. Use (yield (slog-log)) or (yield (writer-log)) instead." + (raise (AttributeError "handler.log removed — use writer_log() or slog_log() inside the effect system"))) (defn is-doeff-handler [handler] diff --git a/tests/effects/test_lazy_ask_env_var_fallback.py b/tests/effects/test_lazy_ask_env_var_fallback.py index b61dd546b..a0fc82b0f 100644 --- a/tests/effects/test_lazy_ask_env_var_fallback.py +++ b/tests/effects/test_lazy_ask_env_var_fallback.py @@ -186,6 +186,6 @@ def program(): "name": "test-service", } - composed = lazy_ask(env=env)(writer()(try_handler(state()(_install_raw_handler(env_var_fallback_handler)(program()))))) + composed = lazy_ask(env=env)(writer(try_handler(state()(_install_raw_handler(env_var_fallback_handler)(program()))))) result = run(scheduled(composed)) assert result == "loaded:/run/secrets/api-key|test-service" diff --git a/tests/effects/test_lazy_ask_inner_handler_propagation.py b/tests/effects/test_lazy_ask_inner_handler_propagation.py index cd9293557..e41db5245 100644 --- a/tests/effects/test_lazy_ask_inner_handler_propagation.py +++ b/tests/effects/test_lazy_ask_inner_handler_propagation.py @@ -115,7 +115,7 @@ def program(): "plain_value": "hello", } - composed = lazy_ask(env=env)(writer()(try_handler(state()(_install_raw_handler(secret_handler)(program()))))) + composed = lazy_ask(env=env)(writer(try_handler(state()(_install_raw_handler(secret_handler)(program()))))) result = run(scheduled(composed)) assert result == "secret:my-api-token|hello" diff --git a/tests/effects/test_memo_no_terminal.py b/tests/effects/test_memo_no_terminal.py index f90392d83..82dba57a4 100644 --- a/tests/effects/test_memo_no_terminal.py +++ b/tests/effects/test_memo_no_terminal.py @@ -26,7 +26,7 @@ def expensive(value): program = _with_handlers( expensive(21), await_handler(), - slog_handler(), + slog_handler, in_memory_memo_handler(), ) diff --git a/tests/effects/test_memo_rewriter_compute_unhandled.py b/tests/effects/test_memo_rewriter_compute_unhandled.py index 8ee4ba751..2e97b4dae 100644 --- a/tests/effects/test_memo_rewriter_compute_unhandled.py +++ b/tests/effects/test_memo_rewriter_compute_unhandled.py @@ -27,7 +27,7 @@ def test_memo_rewriter_does_not_swallow_compute_path_unhandled_effect(): def program(): return (yield Lookup("alpha")) - wrapped = _with_handlers(program(), slog_handler(), rewriter) + wrapped = _with_handlers(program(), slog_handler, rewriter) with pytest.raises(UnhandledEffect): run(wrapped) diff --git a/tests/effects/test_memo_rewriter_no_terminal.py b/tests/effects/test_memo_rewriter_no_terminal.py index dd61315f3..de59e59d8 100644 --- a/tests/effects/test_memo_rewriter_no_terminal.py +++ b/tests/effects/test_memo_rewriter_no_terminal.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from doeff_core_effects.handlers import await_handler, slog_handler +from doeff_core_effects.handlers import await_handler, slog_handler, state from doeff_core_effects.memo_handlers import in_memory_memo_handler, make_memo_rewriter from doeff_core_effects.scheduler import scheduled @@ -46,7 +46,8 @@ def program(): wrapped = _with_handlers( program(), await_handler(), - slog_handler(), + state(), + slog_handler, _lookup_handler(calls), in_memory_memo_handler(), rewriter, @@ -68,7 +69,8 @@ def program(): wrapped = _with_handlers( program(), - slog_handler(), + state(), + slog_handler, _lookup_handler(calls), rewriter, ) diff --git a/tests/effects/test_unhandled_effect_chain.py b/tests/effects/test_unhandled_effect_chain.py index 093258d29..91d362060 100644 --- a/tests/effects/test_unhandled_effect_chain.py +++ b/tests/effects/test_unhandled_effect_chain.py @@ -72,7 +72,7 @@ def prog(): # lazy_ask defaults to Pass-on-miss (PR D), so Ask bubbles through # telemetry_handler → slog_handler → lazy_ask before going Unhandled. - composed = lazy_ask(env={})(slog_handler()(_install_raw_handler(telemetry_handler)(prog()))) + composed = lazy_ask(env={})(slog_handler(_install_raw_handler(telemetry_handler)(prog()))) with pytest.raises(UnhandledEffect) as excinfo: run(composed) msg = _extract_msg(excinfo.value) diff --git a/tests/test_core_effects.py b/tests/test_core_effects.py index c78fb1013..95506c405 100644 --- a/tests/test_core_effects.py +++ b/tests/test_core_effects.py @@ -1,7 +1,15 @@ """Tests for core effects — Ask, Get, Put, Tell.""" from doeff_core_effects.effects import Ask, Get, Put, Slog, Tell, Try -from doeff_core_effects.handlers import reader, slog_handler, state, try_handler, writer +from doeff_core_effects.handlers import ( + reader, + slog_handler, + slog_log, + state, + try_handler, + writer, + writer_log, +) from doeff import Pure, do from doeff import run as doeff_run @@ -68,17 +76,14 @@ def body(): class TestWriter: def test_tell_collects_messages(self): - w = writer() - @do def body(): yield Tell("hello") yield Tell("world") - return "done" + return (yield writer_log()) - result = doeff_run(w(body())) - assert result == "done" - assert w.log == ["hello", "world"] + result = doeff_run(state()(writer(body()))) + assert result == ["hello", "world"] class TestComposed: @@ -97,18 +102,19 @@ def body(): def test_all_three(self): """Reader + State + Writer composed.""" - w = writer() @do def body(): name = yield Ask("name") yield Tell(f"hello {name}") yield Put("greeted", True) - return (yield Get("greeted")) + greeted = yield Get("greeted") + log = yield writer_log() + return (greeted, log) - prog = reader(env={"name": "Bob"})(state()(w(body()))) - assert doeff_run(prog) is True - assert w.log == ["hello Bob"] + prog = reader(env={"name": "Bob"})(state()(writer(body()))) + result = doeff_run(prog) + assert result == (True, ["hello Bob"]) class TestTry: @@ -156,19 +162,17 @@ def body(): class TestSlog: def test_slog_basic(self): - sh = slog_handler() - @do def body(): yield Slog("hello") yield Slog("event", user="alice", action="login") - return "done" + log = yield slog_log() + return log - result = doeff_run(sh(body())) - assert result == "done" - assert len(sh.log) == 2 - assert sh.log[0] == {"msg": "hello"} - assert sh.log[1] == {"msg": "event", "user": "alice", "action": "login"} + result = doeff_run(state()(slog_handler(body()))) + assert len(result) == 2 + assert result[0] == {"msg": "hello"} + assert result[1] == {"msg": "event", "user": "alice", "action": "login"} class TestAwait: diff --git a/tests/test_do_bang_setv.py b/tests/test_do_bang_setv.py index d053abc13..0e5ede9eb 100644 --- a/tests/test_do_bang_setv.py +++ b/tests/test_do_bang_setv.py @@ -7,14 +7,14 @@ import hy # noqa: F401 - activates Hy import hooks for macro tests from doeff_core_effects import slog -from doeff_core_effects.handlers import lazy_ask, writer +from doeff_core_effects.handlers import lazy_ask, state, writer from doeff_core_effects.scheduler import scheduled from doeff import do, run def _run_program(program): - composed = lazy_ask(env={})(writer()(program)) + composed = lazy_ask(env={})(state()(writer(program))) return run(scheduled(composed)) diff --git a/tests/test_docs_withhandler_contract.py b/tests/test_docs_withhandler_contract.py index fcfb9ba42..766e1129f 100644 --- a/tests/test_docs_withhandler_contract.py +++ b/tests/test_docs_withhandler_contract.py @@ -65,7 +65,7 @@ def counter_program(): with warnings.catch_warnings(record=True) as captured: warnings.simplefilter("always", DeprecationWarning) program = counter_program() - program = writer()(program) + program = writer(program) program = state()(program) program = reader(env={"greeting": "hello"})(program) result = run(program) diff --git a/tests/test_double_resume_traceback.py b/tests/test_double_resume_traceback.py index 8059e0739..bc3fd746c 100644 --- a/tests/test_double_resume_traceback.py +++ b/tests/test_double_resume_traceback.py @@ -49,7 +49,7 @@ def program_single_ping() -> EffectGenerator[int]: def _run(program): - wrapped = writer()(try_handler(state()(_install_raw_handler(double_resume_handler)(program)))) + wrapped = writer(try_handler(state()(_install_raw_handler(double_resume_handler)(program)))) return run(scheduled(wrapped)) diff --git a/tests/test_get_handlers_defk.py b/tests/test_get_handlers_defk.py index 8be708746..be0cd7f0d 100644 --- a/tests/test_get_handlers_defk.py +++ b/tests/test_get_handlers_defk.py @@ -109,8 +109,8 @@ def task(): return (yield Wait(t)) composed = _compose(prog(), - writer(), try_handler, state(), await_handler(), - slog_handler(), + writer, try_handler, state(), await_handler(), + slog_handler, marker_handler, ) result = run(scheduled(composed)) @@ -166,8 +166,8 @@ def task(): @do def defk_interpreter(): composed = _compose(prog(), - writer(), try_handler, state(), await_handler(), - slog_handler(), + writer, try_handler, state(), await_handler(), + slog_handler, marker_handler, ) result = yield composed diff --git a/tests/test_memo_rewriter_spawn_continuation.py b/tests/test_memo_rewriter_spawn_continuation.py index fc5b09f8c..2979d72a5 100644 --- a/tests/test_memo_rewriter_spawn_continuation.py +++ b/tests/test_memo_rewriter_spawn_continuation.py @@ -109,11 +109,11 @@ def prog() -> EffectGenerator[dict]: result = _run_scheduled(_compose( prog(), - writer(), + writer, try_handler, state(), await_handler(), - slog_handler(), + slog_handler, simple_fetch_handler, )) assert result == {"close": 1234.5, "ticker": "8801.T", "date": "2026-04-09"} @@ -126,11 +126,11 @@ def prog() -> EffectGenerator[dict]: result = _run_scheduled(_compose( prog(), - writer(), + writer, try_handler, state(), await_handler(), - slog_handler(), + slog_handler, spawn_fetch_handler, )) assert result == {"close": 1234.5, "ticker": "8801.T", "date": "2026-04-09"} diff --git a/tests/test_spawn_with_handlers.py b/tests/test_spawn_with_handlers.py index 2ecbfbfbc..8cf5e45be 100644 --- a/tests/test_spawn_with_handlers.py +++ b/tests/test_spawn_with_handlers.py @@ -51,9 +51,9 @@ def _run_with_handlers(program): handlers = [ reader(env={"model": "test"}), state(), - writer(), + writer, try_handler, - slog_handler(), + slog_handler, custom_query_handler, ] for h in reversed(handlers): diff --git a/tests/test_try_deep_in_handler_body.py b/tests/test_try_deep_in_handler_body.py index 6bfb354e6..f14b5ccdd 100644 --- a/tests/test_try_deep_in_handler_body.py +++ b/tests/test_try_deep_in_handler_body.py @@ -116,7 +116,7 @@ def prog(): env = {"api_key": "sk-test", "client": "prebuilt-client"} wrapped = prog() # reader(outer) → state → writer → try_handler → custom_query_handler(inner) - for h in reversed([reader(env=env), state(), writer(), try_handler, custom_query_handler]): + for h in reversed([reader(env=env), state(), writer, try_handler, custom_query_handler]): wrapped = _program_handler(h)(wrapped) result = run(wrapped) @@ -132,7 +132,7 @@ def prog(): env = {"api_key": "sk-fallback"} # no 'client' key wrapped = prog() - for h in reversed([reader(env=env), state(), writer(), try_handler, custom_query_handler]): + for h in reversed([reader(env=env), state(), writer, try_handler, custom_query_handler]): wrapped = _program_handler(h)(wrapped) result = run(wrapped) @@ -148,7 +148,7 @@ def prog(): env = {} # no client, no api_key wrapped = prog() - for h in reversed([reader(env=env), state(), writer(), try_handler, custom_query_handler]): + for h in reversed([reader(env=env), state(), writer, try_handler, custom_query_handler]): wrapped = _program_handler(h)(wrapped) result = run(wrapped) @@ -167,7 +167,7 @@ def prog(): env = {"api_key": "sk-test"} wrapped = prog() - for h in reversed([reader(env=env), state(), writer(), try_handler, custom_query_handler]): + for h in reversed([reader(env=env), state(), writer, try_handler, custom_query_handler]): wrapped = _program_handler(h)(wrapped) result = run(wrapped) diff --git a/tests/test_try_handler_scope.py b/tests/test_try_handler_scope.py index 0dfefbfbc..694c9ab48 100644 --- a/tests/test_try_handler_scope.py +++ b/tests/test_try_handler_scope.py @@ -8,7 +8,7 @@ effects performed inside Try(program) cannot reach that inner handler. """ from doeff_core_effects import Ask, Try, slog -from doeff_core_effects.handlers import reader, try_handler, writer +from doeff_core_effects.handlers import reader, state, try_handler, writer from doeff_vm import EffectBase from doeff import Pass, Resume, do, run @@ -118,8 +118,8 @@ def prog(): env = {"model": "gpt-5.4"} wrapped = prog() - # reader (outer) → writer → try_handler → custom_handler (inner) - for h in reversed([reader(env=env), writer(), try_handler, custom_handler]): + # reader (outer) → state → writer → try_handler → custom_handler (inner) + for h in reversed([reader(env=env), state(), writer, try_handler, custom_handler]): wrapped = _program_handler(h)(wrapped) result = run(wrapped) diff --git a/tests/test_with_observe_visibility.py b/tests/test_with_observe_visibility.py index e1e53f9df..670805d85 100644 --- a/tests/test_with_observe_visibility.py +++ b/tests/test_with_observe_visibility.py @@ -50,7 +50,7 @@ def prog(): return 42 wrapped = prog() - for h in reversed([state(), writer(), slog_handler()]): + for h in reversed([state(), writer, slog_handler]): wrapped = _program_handler(h)(wrapped) wrapped = WithObserve(VMCallable(observer), wrapped) @@ -79,7 +79,7 @@ def prog(): return result wrapped = prog() - for h in reversed([state(), writer(), try_handler, slog_handler(), custom_handler]): + for h in reversed([state(), writer, try_handler, slog_handler, custom_handler]): wrapped = _program_handler(h)(wrapped) wrapped = WithObserve(VMCallable(observer), wrapped)