From f225a4a15371851a26f772b6c293b17d951a0ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=AB=E5=B2=81?= Date: Tue, 12 May 2026 02:33:57 +0800 Subject: [PATCH 1/2] test(tokenizer): skip tiktoken network-dependent checks when encoding download is forbidden --- llm/tokenizer/tiktoken_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/llm/tokenizer/tiktoken_test.go b/llm/tokenizer/tiktoken_test.go index 5a3009b7..fcb0e928 100644 --- a/llm/tokenizer/tiktoken_test.go +++ b/llm/tokenizer/tiktoken_test.go @@ -1,6 +1,7 @@ package tokenizer import ( + "strings" "testing" "github.com/pkoukk/tiktoken-go" @@ -67,6 +68,18 @@ func TestNewTiktokenTokenizer(t *testing.T) { } } + + +func requireTiktokenAvailable(t *testing.T, tok *TiktokenTokenizer) { + t.Helper() + if _, err := tok.CountTokens("health-check"); err != nil { + if strings.Contains(err.Error(), "Forbidden") || strings.Contains(err.Error(), "403") { + t.Skipf("tiktoken encoding download unavailable in current environment: %v", err) + } + require.NoError(t, err) + } +} + func TestTiktokenTokenizer_PrefixMatch(t *testing.T) { // "gpt-4o-mini" should match "gpt-4o" prefix tok, err := NewTiktokenTokenizer("gpt-4o-mini") @@ -80,6 +93,8 @@ func TestTiktokenTokenizer_CountTokens(t *testing.T) { tok, err := NewTiktokenTokenizer("gpt-4") require.NoError(t, err) + requireTiktokenAvailable(t, tok) + count, err := tok.CountTokens("Hello, world!") require.NoError(t, err) @@ -91,6 +106,8 @@ func TestTiktokenTokenizer_Encode_Decode(t *testing.T) { tok, err := NewTiktokenTokenizer("gpt-4") require.NoError(t, err) + requireTiktokenAvailable(t, tok) + text := "Hello, world!" tokens, err := tok.Encode(text) @@ -107,6 +124,8 @@ func TestTiktokenTokenizer_CountMessages(t *testing.T) { tok, err := NewTiktokenTokenizer("gpt-4") require.NoError(t, err) + requireTiktokenAvailable(t, tok) + messages := []Message{ {Role: "user", Content: "Hello"}, @@ -122,6 +141,8 @@ func TestTiktokenTokenizer_CountMessages(t *testing.T) { func TestTiktokenTokenizer_Name(t *testing.T) { tok, err := NewTiktokenTokenizer("gpt-4") require.NoError(t, err) + requireTiktokenAvailable(t, tok) + assert.Contains(t, tok.Name(), "tiktoken") assert.Contains(t, tok.Name(), "cl100k_base") } @@ -129,6 +150,8 @@ func TestTiktokenTokenizer_Name(t *testing.T) { func TestTiktokenTokenizer_MaxTokens(t *testing.T) { tok, err := NewTiktokenTokenizer("gpt-4") require.NoError(t, err) + requireTiktokenAvailable(t, tok) + assert.Equal(t, 8192, tok.MaxTokens()) } From f8cc00f0d09b71f6d04c5948702552fcf8453910 Mon Sep 17 00:00:00 2001 From: BaSui Date: Thu, 14 May 2026 00:46:45 +0800 Subject: [PATCH 2/2] fix: gofmt tiktoken_test.go --- llm/tokenizer/tiktoken_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/llm/tokenizer/tiktoken_test.go b/llm/tokenizer/tiktoken_test.go index fcb0e928..faee769d 100644 --- a/llm/tokenizer/tiktoken_test.go +++ b/llm/tokenizer/tiktoken_test.go @@ -68,8 +68,6 @@ func TestNewTiktokenTokenizer(t *testing.T) { } } - - func requireTiktokenAvailable(t *testing.T, tok *TiktokenTokenizer) { t.Helper() if _, err := tok.CountTokens("health-check"); err != nil { @@ -95,7 +93,6 @@ func TestTiktokenTokenizer_CountTokens(t *testing.T) { require.NoError(t, err) requireTiktokenAvailable(t, tok) - count, err := tok.CountTokens("Hello, world!") require.NoError(t, err) assert.Greater(t, count, 0) @@ -108,7 +105,6 @@ func TestTiktokenTokenizer_Encode_Decode(t *testing.T) { require.NoError(t, err) requireTiktokenAvailable(t, tok) - text := "Hello, world!" tokens, err := tok.Encode(text) require.NoError(t, err) @@ -126,7 +122,6 @@ func TestTiktokenTokenizer_CountMessages(t *testing.T) { require.NoError(t, err) requireTiktokenAvailable(t, tok) - messages := []Message{ {Role: "user", Content: "Hello"}, {Role: "assistant", Content: "Hi there"},