tests.txt

LLMConfigPrimitives
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  item 0 llm:active => "openai"
  member? "openai" llm:providers-all => true
  empty? (llm:provider-help "openai") => false
  empty? llm:config => false
  is-string? llm:list-models => true

LLMProvidersAll
  extensions [llm]
  length llm:providers-all => 7
  member? "openai" llm:providers-all => true
  member? "anthropic" llm:providers-all => true
  member? "gemini" llm:providers-all => true
  member? "ollama" llm:providers-all => true
  member? "openrouter" llm:providers-all => true
  member? "together" llm:providers-all => true
  member? "groq" llm:providers-all => true

LLMProviderStatus
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  length llm:provider-status => 7

LLMListModels
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  is-string? llm:list-models => true
  empty? llm:list-models => false

LLMActiveConfig
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  item 0 llm:active => "openai"

LLMConfigSummary
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  is-string? llm:config => true
  empty? llm:config => false

LLMHistoryManagement
  extensions [llm]
  globals [history]
  O> llm:clear-history
  llm:history => []
  O> llm:set-history [["user" "hello"] ["assistant" "hi"]]
  O> set history llm:history
  length history => 2
  (item 0 (item 0 history)) => "user"
  (item 1 (item 0 history)) => "hello"
  (item 0 (item 1 history)) => "assistant"
  (item 1 (item 1 history)) => "hi"
  O> llm:clear-history
  llm:history => []

LLMSetModel
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "gpt-4o"
  item 1 llm:active => "gpt-4o"

LLMThinkingConfigPrimitives
  extensions [llm]
  O> llm:set-thinking true
  O> llm:set-reasoning-effort "high"
  O> llm:set-thinking-budget 4096
  O> llm:set-thinking false

LLMReasoningEffortValidation
  extensions [llm]
  O> llm:set-reasoning-effort "none"
  O> llm:set-reasoning-effort "low"
  O> llm:set-reasoning-effort "medium"
  O> llm:set-reasoning-effort "high"
  O> llm:set-reasoning-effort "xhigh"

LLMReasoningEffortIsProviderAware
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "groq"
  O> llm:set-reasoning-effort "low"
  O> llm:set-reasoning-effort "high"
  O> llm:set-reasoning-effort "xhigh" => ERROR Extension exception: Invalid reasoning effort: 'xhigh' for provider 'groq'. Must be one of: high, low, medium
  O> llm:set-reasoning-effort "none" => ERROR Extension exception: Invalid reasoning effort: 'none' for provider 'groq'. Must be one of: high, low, medium
  O> llm:set-provider "openai"
  O> llm:set-reasoning-effort "xhigh"
  O> llm:set-reasoning-effort "none"

LLMThinkingBudgetValidation
  extensions [llm]
  O> llm:set-thinking-budget 1024
  O> llm:set-thinking-budget 8192

LLMChat
  extensions [llm]
  globals [response]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> set response llm:chat "Say hello"
  is-string? response => true
  empty? response => false

LLMChatAsync
  extensions [llm]
  globals [promise result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> set promise llm:chat-async "Say hi"
  is-anonymous-reporter? promise => true
  O> set result (runresult promise)
  is-string? result => true
  empty? result => false

LLMChatHistory
  extensions [llm]
  globals [r1 r2]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set r1 llm:chat "My name is TestBot"
  O> set r2 llm:chat "What is my name?"
  length llm:history => 4
  (item 0 (item 0 llm:history)) => "user"
  (item 0 (item 1 llm:history)) => "assistant"

LLMChooseExactMatch
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  member? (llm:choose "Pick one" ["apple" "banana" "cherry"]) ["apple" "banana" "cherry"] => true

LLMChooseCaseInsensitive
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  (llm:choose "Pick one __TEST_RESPOND:APPLE" ["apple" "banana"]) => "apple"

LLMChooseEmptyListThrows
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> carefully [let __x llm:choose "Pick" []] [set err error-message]
  is-string? err => true

LLMChooseNoMatchThrows
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> carefully [let __x llm:choose "Pick one __TEST_RESPOND:zzzz" ["apple" "banana"]] [set err error-message]
  is-string? err => true

LLMChooseHistoryClean
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:choose "Pick a fruit" ["apple" "banana" "cherry"]
  length llm:history => 2
  (item 0 (item 0 llm:history)) => "user"
  (item 1 (item 0 llm:history)) => "Pick a fruit"
  (item 0 (item 1 llm:history)) => "assistant"
  member? (item 1 (item 1 llm:history)) ["apple" "banana" "cherry"] => true

LLMChooseHistoryUnchangedOnFailure
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [let __x llm:choose "Pick __TEST_RESPOND:zzzz" ["apple" "banana"]] [set err error-message]
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "prior"

LLMChooseThinkingModel
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:choose "Pick one __TEST_THINKING:apple" ["apple" "banana"]
  result => "apple"

LLMChatHistoryCorrectOnSuccess
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat "hello"
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "hello"
  (item 0 (item 1 llm:history)) => "assistant"

LLMTemplateChat
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> set result llm:chat-with-template "demos/simple-template.yaml" [["text" "hello world"]]
  is-string? result => true
  empty? result => false

LLMTemplateWithHistory
  extensions [llm]
  globals [msg1 template-result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set msg1 llm:chat "previous message"
  O> set template-result llm:chat-with-template "demos/simple-template.yaml" [["text" "with history"]]
  is-string? template-result => true
  empty? template-result => false
  length llm:history => 4

LLMChatWithThinking
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-thinking true
  O> set result llm:chat-with-thinking "What is 2 + 2?"
  is-list? result => true
  length result => 2
  is-string? (item 0 result) => true
  is-string? (item 1 result) => true
  empty? (item 0 result) => false

LLMChatWithThinkingBudget
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-thinking true
  O> llm:set-thinking-budget 2048
  O> set result llm:chat-with-thinking "What is 3 + 5?"
  is-list? result => true
  length result => 2
  is-string? (item 0 result) => true
  empty? (item 0 result) => false

LLMMultiAgentChat
  extensions [llm]
  turtles-own [agent-result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> create-turtles 2
  O> ask turtle 0 [ set agent-result llm:chat "Say one" ]
  O> ask turtle 1 [ set agent-result llm:chat "Say two" ]
  is-string? [agent-result] of turtle 0 => true
  is-string? [agent-result] of turtle 1 => true
  empty? [agent-result] of turtle 0 => false
  empty? [agent-result] of turtle 1 => false

LLMOpenRouterProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openrouter"
  item 0 llm:active => "openrouter"
  item 1 llm:active => "openai/gpt-4o-mini"
  empty? (llm:provider-help "openrouter") => false
  O> llm:set-model "openai/gpt-4o"
  item 1 llm:active => "openai/gpt-4o"

LLMTogetherProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "together"
  item 0 llm:active => "together"
  item 1 llm:active => "meta-llama/Llama-3.3-70B-Instruct-Turbo"
  empty? (llm:provider-help "together") => false
  O> llm:set-model "deepseek-ai/DeepSeek-R1"
  item 1 llm:active => "deepseek-ai/DeepSeek-R1"

LLMGroqProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "groq"
  item 0 llm:active => "groq"
  item 1 llm:active => "openai/gpt-oss-20b"
  empty? (llm:provider-help "groq") => false
  O> llm:set-model "openai/gpt-oss-120b"
  item 1 llm:active => "openai/gpt-oss-120b"

LLMGroqChat
  extensions [llm]
  globals [response]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "groq"
  O> llm:clear-history
  O> set response llm:chat "Say hello"
  is-string? response => true
  empty? response => false
  length llm:history => 2

LLMConfigLoading
  extensions [llm]
  O> llm:load-config "demos/config"
  item 0 llm:active => "openai"
  member? "openai" llm:providers => true

LLMLoadConfigRejectsUnknownProvider
  extensions [llm]
  globals [err]
  O> carefully [llm:load-config "demos/test-config-bad-provider"] [set err error-message]
  is-string? err => true
  (position "opanai" err) != false => true

LLMChatHistoryUnchangedOnFailure
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [let __x llm:chat "__TEST_FAIL"] [set err error-message]
  is-string? err => true
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "prior"

LLMChatWithThinkingHistoryUnchangedOnFailure
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [let __x llm:chat-with-thinking "__TEST_FAIL"] [set err error-message]
  is-string? err => true
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "prior"

LLMChatAsyncOverlappingPairsAtomic
  extensions [llm]
  globals [p1 p2 r1 r2 h]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set p1 llm:chat-async "__TEST_DELAY:300:slow"
  O> set p2 llm:chat-async "fast"
  O> set r1 (runresult p1)
  O> set r2 (runresult p2)
  r1 => "stub:slow"
  r2 => "stub:fast"
  O> set h llm:history
  length h => 4
  (item 0 (item 0 h)) => "user"
  (item 0 (item 1 h)) => "assistant"
  (item 0 (item 2 h)) => "user"
  (item 0 (item 3 h)) => "assistant"
  member? (item 1 (item 1 h)) ["stub:slow" "stub:fast"] => true
  member? (item 1 (item 3 h)) ["stub:slow" "stub:fast"] => true

LLMMixedSyncAsyncHistoryAtomic
  extensions [llm]
  globals [p1 r1 r2 h]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set p1 llm:chat-async "__TEST_DELAY:300:slow"
  O> set r1 llm:chat "quick"
  O> set r2 (runresult p1)
  O> set h llm:history
  length h => 4
  (item 0 (item 0 h)) => "user"
  (item 0 (item 1 h)) => "assistant"
  (item 0 (item 2 h)) => "user"
  (item 0 (item 3 h)) => "assistant"

LLMCompileErrorValid
  extensions [llm]
  llm:compile-error "fd 1" => ""
  llm:compile-error "rt 90 fd 1" => ""
  llm:compile-error "" => ""
  llm:compile-error "   " => ""
  llm:compile-error "ifelse xcor > 0 [ fd 1 ] [ bk 1 ]" => ""
  llm:compile-error "rt (random 90) * 0.5 + (random 45) * 0.5" => ""

LLMCompileErrorInvalid
  extensions [llm]
  (llm:compile-error "frobnicate") != "" => true
  (llm:compile-error "fd") != "" => true
  (llm:compile-error "if xcor > 0 [fd 1] [bk 1]") != "" => true
  (llm:compile-error "ifelse xcor > 0 [ fd 1 ]") != "" => true
  (llm:compile-error "rt undefined-var-xyz") != "" => true
  is-string? (llm:compile-error "frobnicate") => true

LLMCompileErrorTurtleContext
  extensions [llm]
  llm:compile-error "set heading 90" => ""
  llm:compile-error "set color red fd 1" => ""


LLMCompileErrorLEARCorpus
  extensions [llm]
  llm:compile-error "ifelse random 2 < 1 [lt 10 fd 1] [ifelse random 2 < 1 [rt 10 fd 1] [fd 1] ]" => ""
  llm:compile-error "ifelse random 2 = 0 [ lt random 20 fd 1 ] [ rt random 20 fd 1 ]" => ""
  llm:compile-error "ifelse random 2 = 0 [fd 1] [rt 90 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 10 fd 1] [rt random 10 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 20 fd 1] [rt random 10 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 30 fd 1] [rt random 30 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [rt random 30 fd 1] [lt random 30 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [ifelse random 2 = 0 [lt 10 fd 1] [rt 10 fd 1]] [ifelse random 2 = 0 [lt random 10 fd 1] [rt random 10 fd 1]]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 10 fd 1] [ifelse random 2 = 0 [rt random 10 fd 1] [fd 1] ]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 10 fd 1] [ifelse random 2 = 1 [rt random 10 fd 1] [fd 1] ]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 20 fd 1] [rt random 20 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random-float 10 fd 1] [rt random-float 10 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [ fd 1 ] [ ifelse random 2 = 1 [ rt 10 fd 1 ] [ lt 10 fd 1 ] ]" => ""
  llm:compile-error "ifelse random-float 1 < 0.4 [lt random 10 fd 1] [ifelse random-float 1 < 0.3 [rt random 10 fd 1] [fd 1] ]" => ""

LLMCompileErrorBannedPrimitives
  extensions [llm]
  (llm:compile-error "fd 1" ["die"]) => ""
  (llm:compile-error "fd 1" []) => ""
  (llm:compile-error "die" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "fd 1 die" ["die" "clear-all"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "DIE" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "die" ["DIE"]) => "Disallowed primitive(s) used: die"

LLMCompileErrorBannedNoFalsePositives
  extensions [llm]
  (llm:compile-error "fd 1 ; die is banned" ["die"]) => ""
  (llm:compile-error "let diehard 1 fd diehard" ["die"]) => ""
  (llm:compile-error "print \"die\"" ["die"]) => ""

LLMCompileErrorSyntaxBeforeBanned
  extensions [llm]
  (llm:compile-error "frobnicate die" ["die"]) != "" => true
  member? "Disallowed" (llm:compile-error "frobnicate die" ["die"]) => false

LLMCompileErrorMultiLineLocals
  extensions [llm]
  llm:compile-error "let x 5\nfd x" => ""
  llm:compile-error "let x 5\nlet y x * 2\nrt y\nfd 1" => ""
  llm:compile-error "let x 5\n\n  rt x\n  fd 1\n" => ""
  llm:compile-error "let d 3\nifelse d > 2\n  [ fd d ]\n  [ rt 90 fd 1 ]" => ""
  llm:compile-error "  \n  fd 1  \n  " => ""
  llm:compile-error "let x 1 let y 2 let z 3 fd x + y + z" => ""
  (llm:compile-error "let x 5\nfd undefined-thing") != "" => true
  (llm:compile-error "fd x\nlet x 5") != "" => true
  (llm:compile-error "let x 5\nlet x 6\nfd x") != "" => true
  (llm:compile-error "let x 5\nlet x 6\nfd x") => "There is already a local variable here called X (offset 43)"

LLMCompileErrorTurtleOwnVariables
  extensions [llm]
  llm:compile-error "fd xcor" => ""
  llm:compile-error "set heading heading + 10 fd 1" => ""
  llm:compile-error "ifelse pcolor = green [ fd 1 ] [ rt 90 ]" => ""
  llm:compile-error "let n count turtles in-radius 3\nfd n" => ""
  (llm:compile-error "fd not-a-turtle-variable") != "" => true

LLMCompileErrorComments
  extensions [llm]
  llm:compile-error "; only a comment" => ""
  llm:compile-error "fd 1 ; move forward" => ""
  llm:compile-error "; leading comment\nfd 1\n; trailing comment" => ""
  llm:compile-error "let x 5 ; set speed\nfd x" => ""

LLMCompileErrorNestedAndBlocks
  extensions [llm]
  llm:compile-error "ask other turtles in-radius 2 [ rt 180 ]" => ""
  llm:compile-error "repeat 3 [ fd 1 rt 120 ]" => ""
  llm:compile-error "if any? other turtles [ fd 1 ]" => ""
  llm:compile-error "(ifelse xcor > 5 [ rt 90 ] xcor < -5 [ lt 90 ] [ fd 1 ])" => ""
  llm:compile-error "foreach [1 2 3] [ n -> fd n ]" => ""
  (llm:compile-error "repeat 3 [ fd 1 rt 120") != "" => true
  (llm:compile-error "ask turtles [ fd 1 ] ]") != "" => true

LLMCompileErrorBannedMultiLine
  extensions [llm]
  (llm:compile-error "let x 5\nfd x\ndie" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "fd 1\n; die here is a comment\nrt 90" ["die"]) => ""
  (llm:compile-error "ask turtles [ die ]" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "fd 1\nrt 90" ["die" "clear-all" "hatch"]) => ""
  (llm:compile-error "let diehard 1\nfd diehard" ["die"]) => ""

LLMCompileErrorBannedMultipleFound
  extensions [llm]
  (llm:compile-error "ask turtles [ die ]" ["die" "ask"]) => "Disallowed primitive(s) used: ask, die"
  (llm:compile-error "die die die" ["die"]) => "Disallowed primitive(s) used: die"

LLMCompileErrorBannedEdgeCases
  extensions [llm]
  (llm:compile-error "" ["die"]) => ""
  (llm:compile-error "fd 1" ["  "]) => ""
  (llm:compile-error "fd 1" ["fd"]) => "Disallowed primitive(s) used: fd"
  (llm:compile-error "fd 1" ["  die  "]) => ""
  (llm:compile-error "die" ["  die  "]) => "Disallowed primitive(s) used: die"

LLMCompileErrorBannedListValidation
  extensions [llm]
  (llm:compile-error "die" [1 2]) => ERROR Extension exception: llm:compile-error expects a list of primitive names as strings, but got: 1, 2
  (llm:compile-error "fd 1" ["die" 5]) => ERROR Extension exception: llm:compile-error expects a list of primitive names as strings, but got: 5