Unknown fields, native records, and 0.4.0

CHANGELOG.md

@@ -1,6 +1,33 @@
 # Changelog
 
-## Unreleased
+## v0.4.0 - 2026-10-23
+
+- Add the `:unknown_fields` Rustler generator option. `:runtime_schema` makes
+  generated message decoders skip fields the compiled schema does not define,
+  using a `KiwiRuntimeSchema` read from the schema that accompanies the data, so
+  data written with a newer, compatible schema decodes instead of failing with
+  `BadArg`. The default, `:error`, keeps rejecting such fields.
+- Route every generated message decoder's unknown fields through one
+  `kiwi_unknown_field` helper, and pass full message decoders their definition
+  name.
+- Accept a `:schema` option in `KiwiCodec.decode/3` to skip message fields the
+  generated modules do not define, using the schema the data was written with.
+- Add `KiwiCodec.SchemaInterpreter.skip_message_field/4`.
+- Add the `:records` Rustler generator feature. It decodes Kiwi data into
+  native records Rust code can read, change, and share without building Elixir
+  terms, then encodes them as the sparse maps generated decoders return,
+  optionally only some fields. Generated constants name each definition, field,
+  and enum value; records have typed getters and helpers that update nested
+  records and scalar, string, enum, and float-array fields. See
+  `KiwiCodec.RustlerGenerator.Records`.
+- Sparse decoders intern their module and field-name atoms once instead of for
+  every map they build, which took two thirds of the time decoding large files.
+- Require RustQ 1.0.0-rc.13 for code generation.
+- Rename the Rustler generator's skip, sparse, and full decoder helpers by
+  family: `RustlerGenerator.SkipHelpers`, `SkipValueHelpers`, and
+  `SkipDescriptor` are `RustlerGenerator.Skip.Helpers`, `Skip.ValueHelpers`,
+  and `Skip.Descriptor`; `SparseHelpers` is `Sparse.Helpers`; and
+  `FullDecoderHelpers` is `Full.Helpers`.
 
 ## v0.3.1 - 2026-07-18
 

README.md

@@ -199,7 +199,7 @@ Once published, add it to your dependencies:
 ```elixir
 def deps do
   [
-    {:kiwi_codec, "~> 0.3.0"}
+    {:kiwi_codec, "~> 0.4.0"}
   ]
 end
 ```

codegen/kiwi_codec/rustler_generator.ex

@@ -13,6 +13,7 @@ defmodule KiwiCodec.RustlerGenerator do
 
   alias KiwiCodec.RustlerGenerator.Definition
   alias KiwiCodec.RustlerGenerator.Entrypoint
+  alias KiwiCodec.RustlerGenerator.Records
   alias KiwiCodec.RustlerGenerator.Selection
   alias KiwiCodec.RustlerGenerator.Skip
   alias KiwiCodec.RustlerGenerator.Sparse
@@ -54,7 +55,13 @@ defmodule KiwiCodec.RustlerGenerator do
       `defrust` and RustQ uses the real decoder method metadata for propagation.
     * `:features` - decoder families to generate. Defaults to `[:full]`.
       Use `:sparse` and `:skip` for generic sparse map decoders and skip
-      helpers used by projection-oriented native backends.
+      helpers used by projection-oriented native backends, and `:records` for
+      native records Rust code can work with before encoding them; see
+      `KiwiCodec.RustlerGenerator.Records`.
+    * `:unknown_fields` - how message decoders handle fields the schema does
+      not define: `:error` (default) rejects the data; `:runtime_schema` skips
+      them using a schema read with the data. See
+      `KiwiCodec.RustlerGenerator.UnknownFields` for the `Decoder` contract.
     * `:sparse_messages` - sparse message generation strategy. Defaults to
       `:match`. Use `:descriptor` to generate compact descriptor-backed sparse
       message decoders and benchmark before adopting on hot schemas.
@@ -106,6 +113,9 @@ defmodule KiwiCodec.RustlerGenerator do
             )
         )
 
+      :records ->
+        Records.fragments(selected, module_prefix)
+
       :skip ->
         Skip.fragments(selected, definition_map,
           messages?: not shared_sparse_skip?,

codegen/kiwi_codec/rustler_generator/atom_cache.ex

@@ -0,0 +1,29 @@
+defmodule KiwiCodec.RustlerGenerator.AtomCache do
+  use RustQ.Meta
+
+  @moduledoc """
+  Interns the atoms generated sparse decoders put in every map: the module name, the
+  `__kiwi_module__` key, and each field name.
+
+  Creating an atom from its name takes the VM's atom table lock and hashes the text, which
+  dominated decoding large files. `kiwi_atom(env, name)` does it once per name and scheduler
+  thread.
+  """
+
+  alias RustQ.Type, as: R
+
+  @template Path.expand("templates/atom_cache.rs", __DIR__)
+  @external_resource @template
+
+  @doc "Returns the Rust items defining `kiwi_atom`."
+  @spec fragments() :: [RustQ.Rust.Fragment.t()]
+  def fragments do
+    [RustQ.Rust.fragment(:item, @template |> File.read!() |> String.trim_trailing())]
+  end
+
+  # Typed declaration for callers; the emitted definition is the template.
+  @spec kiwi_atom(R.path(:Env, R.lifetime(:_)), R.raw(:"&'static str")) :: R.path(:Atom)
+  defrust kiwi_atom(env, name) do
+    Atom.from_str(env, name).unwrap()
+  end
+end

codegen/kiwi_codec/rustler_generator/decoder_macro.ex

@@ -8,7 +8,7 @@ defmodule KiwiCodec.RustlerGenerator.DecoderMacro do
   authoring in Elixir/RustQ while preserving small generated source.
   """
 
-  alias KiwiCodec.RustlerGenerator.FullDecoderHelpers
+  alias KiwiCodec.RustlerGenerator.Full
   alias KiwiCodec.RustlerGenerator.Name
   alias KiwiCodec.RustlerGenerator.RustExpr
   alias KiwiCodec.Schema.Enum, as: SchemaEnum
@@ -17,7 +17,7 @@ defmodule KiwiCodec.RustlerGenerator.DecoderMacro do
 
   @spec enum_decoder(SchemaEnum.t()) :: RustQ.Rust.Fragment.t()
   def enum_decoder(%SchemaEnum{} = definition) do
-    FullDecoderHelpers.macro_call(:kiwi_enum_decoder,
+    Full.Helpers.macro_call(:kiwi_enum_decoder,
       fn: RustExpr.ident(Name.decoder_function(definition.name)),
       env: :env,
       decoder: :decoder,
@@ -31,7 +31,7 @@ defmodule KiwiCodec.RustlerGenerator.DecoderMacro do
   @spec struct_decoder(Struct.t(), String.t(), map(), (term(), map() -> iodata())) ::
           RustQ.Rust.Fragment.t()
   def struct_decoder(%Struct{} = definition, module_prefix, definition_map, field_expr_fun) do
-    FullDecoderHelpers.macro_call(:kiwi_struct_decoder,
+    Full.Helpers.macro_call(:kiwi_struct_decoder,
       fn: RustExpr.ident(Name.decoder_function(definition.name)),
       env: :env,
       decoder: :decoder,
@@ -46,14 +46,15 @@ defmodule KiwiCodec.RustlerGenerator.DecoderMacro do
   @spec message_decoder(Message.t(), String.t(), map(), (term(), map() -> iodata())) ::
           RustQ.Rust.Fragment.t()
   def message_decoder(%Message{} = definition, module_prefix, definition_map, _field_expr_fun) do
-    FullDecoderHelpers.macro_call(:kiwi_message_decoder,
+    Full.Helpers.macro_call(:kiwi_message_decoder,
       fn: RustExpr.ident(Name.decoder_function(definition.name)),
       fields_fn: RustExpr.ident(Name.message_fields_function(definition.name)),
       env: :env,
       decoder: :decoder,
       module_static: RustExpr.ident(Name.module_atom_static(definition.name)),
       keys_static: RustExpr.ident(Name.struct_keys_static(definition.name)),
       module: Name.module_name(module_prefix, definition.name),
+      definition: definition.name,
       keys: key_rows(definition.fields),
       fields: message_field_rows(definition.fields, definition_map)
     )

codegen/kiwi_codec/rustler_generator/full_decoder_helpers.ex → codegen/kiwi_codec/rustler_generator/full/helpers.ex

@@ -1,6 +1,10 @@
-defmodule KiwiCodec.RustlerGenerator.FullDecoderHelpers do
+defmodule KiwiCodec.RustlerGenerator.Full.Helpers do
   use RustQ.Meta,
-    callable_modules: [RustQ.Rustler.Atom, RustQ.Rustler.Term]
+    callable_modules: [
+      RustQ.Rustler.Atom,
+      RustQ.Rustler.Term,
+      KiwiCodec.RustlerGenerator.UnknownFields
+    ]
 
   @moduledoc """
   RustQ-authored full Kiwi decoder macro helpers.
@@ -110,6 +114,7 @@ defmodule KiwiCodec.RustlerGenerator.FullDecoderHelpers do
                  module_static: module_static(:ident),
                  keys_static: keys_static(:ident),
                  module: module_name(:literal),
+                 definition: definition_name(:literal),
                  keys:
                    repeat do
                      key(:literal)
@@ -154,6 +159,7 @@ defmodule KiwiCodec.RustlerGenerator.FullDecoderHelpers do
       kiwi_message_fields(
         env,
         decoder,
+        definition_name,
         struct_keys,
         struct_values,
         array([
@@ -281,11 +287,12 @@ defmodule KiwiCodec.RustlerGenerator.FullDecoderHelpers do
   @spec kiwi_message_fields(
           R.path(:Env, R.lifetime(:a)),
           R.mut_ref(R.path(:Decoder, R.lifetime(:_))),
+          R.str(),
           R.slice(R.path({:rustler, :wrapper, :NIF_TERM})),
           R.vec(R.path({:rustler, :wrapper, :NIF_TERM})),
           R.slice(R.path(:KiwiFullField))
         ) :: R.nif_result(term())
-  defrust kiwi_message_fields(env, decoder, keys, values, fields) do
+  defrust kiwi_message_fields(env, decoder, definition_name, keys, values, fields) do
     field_id = decoder.read_var_uint()
 
     if field_id == 0 do
@@ -297,10 +304,11 @@ defmodule KiwiCodec.RustlerGenerator.FullDecoderHelpers do
           value = kiwi_full_field_value(env, decoder, field)
           next_values = values
           assign!(index(next_values, field.index), value.as_c_arg())
-          kiwi_message_fields(env, decoder, keys, next_values, fields)
+          kiwi_message_fields(env, decoder, definition_name, keys, next_values, fields)
 
         {:error, _index} ->
-          {:error, badarg()}
+          kiwi_unknown_field(decoder, definition_name, field_id)
+          kiwi_message_fields(env, decoder, definition_name, keys, values, fields)
       end
     end
   end

codegen/kiwi_codec/rustler_generator/records.ex

@@ -0,0 +1,140 @@
+defmodule KiwiCodec.RustlerGenerator.Records do
+  @moduledoc """
+  Generates native Kiwi records: decoded values Rust code can read, change, and share without
+  building Elixir terms, and encode later as the sparse maps generated decoders return.
+
+  The record runtime is schema-independent. The schema only supplies a definition table, read
+  by the decoder and encoder, and constants naming each definition, field, and enum value for
+  code that works with records:
+
+      kiwi_record_definitions::NODE_CHANGE
+      kiwi_records::node_change::FILL_PAINTS
+      kiwi_records::node_phase::REMOVED
+
+  Decoding needs the caller's `Decoder<'a>` to provide, besides the primitive readers,
+  `read_byte_array_value(&mut self) -> NifResult<&'a [u8]>`,
+  `read_string_str(&mut self) -> NifResult<&'a str>`, and
+  `remaining(&self) -> usize`. Unknown message fields go through `kiwi_unknown_field`; see
+  `KiwiCodec.RustlerGenerator.UnknownFields`.
+  """
+
+  alias KiwiCodec.RustlerGenerator.Name
+  alias KiwiCodec.Schema.Enum, as: SchemaEnum
+  alias KiwiCodec.Schema.Struct
+  alias RustQ.Rust.AST.Builder, as: A
+
+  @template Path.expand("templates/records_runtime.rs", __DIR__)
+  @external_resource @template
+
+  @primitives ~w(bool byte int uint float string int64 uint64)
+
+  @rust_keywords ~w(as async await box break const continue crate dyn else enum extern false fn
+                    for if impl in let loop match mod move mut pub ref return self static struct
+                    super trait true type unsafe use where while yield)
+
+  @kinds %{SchemaEnum => 0, Struct => 1, KiwiCodec.Schema.Message => 2}
+
+  @doc "Returns the record runtime, definition table, and constants for `definitions`."
+  @spec fragments([KiwiCodec.Schema.definition()], String.t()) :: [term()]
+  def fragments(definitions, module_prefix) do
+    indexes =
+      definitions |> Enum.with_index() |> Map.new(fn {def, index} -> {def.name, index} end)
+
+    [
+      RustQ.Rust.fragment(:item, @template |> File.read!() |> String.trim_trailing()),
+      definition_table(definitions, indexes, module_prefix),
+      constants(definitions, indexes)
+    ]
+  end
+
+  defp definition_table(definitions, indexes, module_prefix) do
+    A.static(
+      :KIWI_RECORD_DEFINITIONS,
+      {:ref, {:slice, :KiwiRecordDefinition}},
+      A.slice(Enum.map(definitions, &definition(&1, indexes, module_prefix)))
+    )
+  end
+
+  defp definition(definition, indexes, module_prefix) do
+    A.struct_expr(:KiwiRecordDefinition,
+      kind: A.lit(Map.fetch!(@kinds, definition.__struct__)),
+      name: A.lit(definition.name),
+      module: A.lit(Name.module_name(module_prefix, definition.name)),
+      fields: A.slice(definition |> fields() |> Enum.map(&field(&1, indexes))),
+      variants: A.slice(definition |> variants() |> Enum.map(&variant/1))
+    )
+  end
+
+  defp field(field, indexes) do
+    A.struct_expr(:KiwiRecordField,
+      id: A.lit(field.id),
+      name: A.lit(Name.field_name(field.name)),
+      type_index: A.lit(type_index(field.type, indexes)),
+      array: A.lit(field.array?)
+    )
+  end
+
+  defp variant(variant) do
+    A.struct_expr(:KiwiRecordVariant,
+      value: A.lit(variant.value),
+      name: A.lit(Macro.underscore(variant.name))
+    )
+  end
+
+  # Primitive types are bitwise-negated indexes, definitions their table index.
+  defp type_index(type, indexes) do
+    case Enum.find_index(@primitives, &(&1 == type)) do
+      nil -> Map.fetch!(indexes, type)
+      primitive -> Bitwise.bnot(primitive)
+    end
+  end
+
+  # Struct fields keep wire order, which their ids follow; message fields sort by id.
+  defp fields(%Struct{fields: fields}), do: fields
+  defp fields(%{fields: fields}), do: Enum.sort_by(fields, & &1.id)
+  defp fields(_enum), do: []
+
+  defp variants(%SchemaEnum{variants: variants}), do: Enum.sort_by(variants, & &1.value)
+  defp variants(_definition), do: []
+
+  # Definition indexes live apart from field and enum constants, which take any schema name.
+  defp constants(definitions, indexes) do
+    [
+      A.module(
+        :kiwi_record_definitions,
+        Enum.map(definitions, fn definition ->
+          name = definition.name |> Name.rust_identifier() |> String.upcase()
+          A.const(name, :u32, A.lit(Map.fetch!(indexes, definition.name)), vis: :pub)
+        end),
+        vis: :pub
+      ),
+      A.module(
+        :kiwi_records,
+        Enum.map(definitions, fn definition ->
+          A.module(
+            module_identifier(definition.name),
+            Enum.map(fields(definition), &field_constant/1) ++
+              Enum.map(variants(definition), &variant_constant/1),
+            vis: :pub
+          )
+        end),
+        vis: :pub
+      )
+    ]
+  end
+
+  defp field_constant(field) do
+    name = field.name |> Name.rust_identifier() |> String.upcase()
+    A.const(name, :u32, A.lit(field.id), vis: :pub)
+  end
+
+  defp variant_constant(variant) do
+    name = variant.name |> Name.rust_identifier() |> String.upcase()
+    A.const(name, :u32, A.lit(variant.value), vis: :pub)
+  end
+
+  defp module_identifier(name) do
+    identifier = Name.rust_identifier(name)
+    if identifier in @rust_keywords, do: identifier <> "_", else: identifier
+  end
+end

codegen/kiwi_codec/rustler_generator/skip.ex

@@ -8,8 +8,8 @@ defmodule KiwiCodec.RustlerGenerator.Skip do
   """
 
   alias KiwiCodec.RustlerGenerator.RustExpr
-  alias KiwiCodec.RustlerGenerator.SkipDescriptor
-  alias KiwiCodec.RustlerGenerator.SkipValueHelpers
+  alias __MODULE__.Descriptor
+  alias __MODULE__.ValueHelpers
   alias KiwiCodec.Schema.Enum, as: SchemaEnum
   alias KiwiCodec.Schema.{Message, Struct}
   alias RustQ.Meta.AST, as: MetaAST
@@ -57,7 +57,7 @@ defmodule KiwiCodec.RustlerGenerator.Skip do
   defp definition(%SchemaEnum{}, _definition_map, _messages?, _struct_mode), do: []
 
   defp definition(%Struct{name: name, fields: fields}, definition_map, _messages?, _struct_mode) do
-    MetaAST.macro_call!(SkipValueHelpers, :kiwi_skip_struct_decoder,
+    MetaAST.macro_call!(ValueHelpers, :kiwi_skip_struct_decoder,
       fn: skip_function_name(name),
       decoder: :decoder,
       fields: skip_descriptor_rows(fields, definition_map)
@@ -67,7 +67,7 @@ defmodule KiwiCodec.RustlerGenerator.Skip do
   defp definition(%Message{}, _definition_map, false, _struct_mode), do: []
 
   defp definition(%Message{name: name, fields: fields}, definition_map, true, _struct_mode) do
-    MetaAST.macro_call!(SkipValueHelpers, :kiwi_skip_message_decoder,
+    MetaAST.macro_call!(ValueHelpers, :kiwi_skip_message_decoder,
       fn: skip_function_name(name),
       decoder: :decoder,
       definition: name,
@@ -127,5 +127,5 @@ defmodule KiwiCodec.RustlerGenerator.Skip do
   end
 
   defp scalar_skip_function(type, definition_map),
-    do: SkipDescriptor.scalar_function(type, definition_map)
+    do: Descriptor.scalar_function(type, definition_map)
 end