From 82eee43e8d65cdf5c892db82538145228bacd603 Mon Sep 17 00:00:00 2001 From: Andrea Leopardi Date: Sun, 12 Jul 2026 13:28:17 +0200 Subject: [PATCH] Support graphemes/codepoints in string/2 Closes #99. --- lib/stream_data.ex | 67 ++++++++++++++++++++++++++++++++++++--- test/stream_data_test.exs | 29 +++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/lib/stream_data.ex b/lib/stream_data.ex index 2ef1b76..687f1e3 100644 --- a/lib/stream_data.ex +++ b/lib/stream_data.ex @@ -1969,7 +1969,24 @@ defmodule StreamData do ## Options - See the documentation of `list_of/2` for the possible values of options. + This function accepts the same length-related options as `list_of/2` + (`:length`, `:min_length`, and `:max_length`), plus: + + * `:count` - (`:codepoints` or `:graphemes`) controls what the + length-related options count. Defaults to `:codepoints`. + + * `:codepoints` - the length-related options count the number of + Unicode codepoints in the generated string. This is what `list_of/2` + counts under the hood, and it is the default for backwards + compatibility. + + * `:graphemes` - the length-related options count the number of + graphemes in the generated string, that is, what `String.length/1` + returns. This matters for kinds such as `:printable` and `:utf8`, + which can generate combining characters: a base character followed + by a combining mark is two codepoints but a single grapheme. Without + this option, `String.length/1` of a generated string can be *smaller* + than the requested minimum length. *Available since 1.4.0.* ## Examples @@ -2019,9 +2036,51 @@ defmodule StreamData do end defp string_from_codepoint_data(codepoint_data, options) do - codepoint_data - |> list_of(options) - |> map(&List.to_string/1) + {count, options} = Keyword.pop(options, :count, :codepoints) + + unless count in [:codepoints, :graphemes] do + raise ArgumentError, + ":count must be either :codepoints or :graphemes, got: #{inspect(count)}" + end + + string = + codepoint_data + |> list_of(options) + |> map(&List.to_string/1) + + # The length-related options are enforced by list_of/2 on the list of + # codepoints. When counting graphemes, a codepoint can combine with the + # previous one into a single grapheme (for example, a combining mark), so + # String.length/1 can end up *below* the requested minimum. Since the number + # of graphemes is always <= the number of codepoints, only the lower bound + # can be violated, so that's all we need to enforce here. + case count do + :codepoints -> + string + + :graphemes -> + case min_grapheme_count(options) do + 0 -> string + min -> filter(string, &(String.length(&1) >= min)) + end + end + end + + defp min_grapheme_count(options) do + case Keyword.fetch(options, :length) do + {:ok, length} when is_integer(length) and length >= 0 -> + length + + {:ok, min..max//_} when min >= 0 and max >= 0 -> + Kernel.min(min, max) + + _other -> + case Keyword.get(options, :min_length, 0) do + min when is_integer(min) and min >= 0 -> min + # Leave invalid values to list_of/2, which raises a proper error. + _invalid -> 0 + end + end end @doc """ diff --git a/test/stream_data_test.exs b/test/stream_data_test.exs index 5ba2ccc..af77e2d 100644 --- a/test/stream_data_test.exs +++ b/test/stream_data_test.exs @@ -762,6 +762,35 @@ defmodule StreamDataTest do assert String.length(string) == 3 end end + + property "with count: :graphemes, the length options count graphemes" do + check all string <- string(:printable, min_length: 41, count: :graphemes) do + assert String.length(string) >= 41 + end + + check all string <- string(:printable, length: 20, count: :graphemes) do + assert String.length(string) == 20 + end + + check all string <- string(:printable, length: 10..15, count: :graphemes) do + assert String.length(string) in 10..15 + end + + check all string <- string(:utf8, min_length: 30, count: :graphemes) do + assert String.length(string) >= 30 + end + end + + property "with count: :codepoints, the length options count codepoints" do + check all string <- string(:printable, min_length: 41, count: :codepoints) do + assert length(String.to_charlist(string)) >= 41 + end + end + + test "with an invalid :count option" do + message = ":count must be either :codepoints or :graphemes, got: :bytes" + assert_raise ArgumentError, message, fn -> string(:printable, count: :bytes) end + end end describe "atom/1" do