From 727860824f12e25d489e3d5bcf2e1ecf9e83b2bb Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 20 Oct 2026 18:23:15 +0200 Subject: [PATCH 1/8] feat(ai-components): add VoiceWave and VoiceGlow voice-agent visualizations Port the voice agent visualizations from stream-chat-android (GetStream/stream-chat-android#6848, #6849, #6851) to the ai-components entry point, with the same motion maths. - VoiceWave: a string carrying a wave packet, drawn on a 2D canvas. - VoiceGlow: a soft globe drawn by a WebGL shader, with a 2D canvas fallback chosen by `rendering`. - useAudioLevel: a pull-based level getter for a MediaStream, for the `audioLevel` prop. - Both follow the theme through the voice-color and voice-backdrop tokens, honour prefers-reduced-motion, pause off screen and are labelled for assistive technology in all 12 locales. The ai-chatbot example gains a hands-free voice mode built on them, and its send path moves into a shared useSendAIMessage hook. --- examples/ai-chatbot/README.md | 8 +- examples/ai-chatbot/implementation.md | 17 + .../ChatContainer/ChatContainer.tsx | 20 +- .../MessageInputBar/MessageInputBar.tsx | 71 +--- .../src/components/VoiceMode/VoiceMode.scss | 111 ++++++ .../src/components/VoiceMode/VoiceMode.tsx | 109 ++++++ .../components/VoiceMode/VoiceModeButton.tsx | 35 ++ .../components/VoiceMode/VoiceModeContext.tsx | 34 ++ .../src/components/VoiceMode/index.ts | 3 + .../VoiceMode/useVoiceConversation.ts | 259 +++++++++++++ examples/ai-chatbot/src/components/index.scss | 3 + .../src/components/useSendAIMessage.ts | 94 +++++ src/i18n/de.json | 6 + src/i18n/en.json | 6 + src/i18n/es.json | 6 + src/i18n/fr.json | 6 + src/i18n/hi.json | 6 + src/i18n/it.json | 6 + src/i18n/ja.json | 6 + src/i18n/ko.json | 6 + src/i18n/nl.json | 6 + src/i18n/pt.json | 6 + src/i18n/ru.json | 6 + src/i18n/tr.json | 6 + src/plugins/AIComponents/README.md | 79 +++- .../VoiceVisualization/VoiceGlow.tsx | 59 +++ .../VoiceVisualizationView.tsx | 267 +++++++++++++ .../VoiceVisualization/VoiceWave.tsx | 30 ++ .../AIComponents/VoiceVisualization/bounce.ts | 56 +++ .../AIComponents/VoiceVisualization/driver.ts | 216 +++++++++++ .../AIComponents/VoiceVisualization/index.ts | 8 + .../AIComponents/VoiceVisualization/math.ts | 68 ++++ .../VoiceVisualization/palette.ts | 91 +++++ .../renderers/glowCanvasRenderer.ts | 214 ++++++++++ .../renderers/glowWebGLRenderer.ts | 365 ++++++++++++++++++ .../VoiceVisualization/renderers/types.ts | 20 + .../renderers/waveRenderer.ts | 123 ++++++ .../AIComponents/VoiceVisualization/types.ts | 55 +++ .../VoiceVisualization/useAudioLevel.ts | 79 ++++ .../__tests__/VoiceVisualization.test.tsx | 259 +++++++++++++ .../AIComponents/__tests__/a11y.test.tsx | 29 +- .../__tests__/classNames.test.tsx | 13 +- .../AIComponents/__tests__/hydration.test.tsx | 37 +- .../AIComponents/__tests__/i18n.test.tsx | 27 +- .../AIComponents/__tests__/ssr.test.tsx | 19 +- .../__tests__/styleTokens.test.ts | 12 + .../__tests__/useAudioLevel.test.ts | 110 ++++++ .../__tests__/voiceDriver.test.ts | 238 ++++++++++++ .../__tests__/voicePalette.test.ts | 83 ++++ src/plugins/AIComponents/index.ts | 1 + src/plugins/AIComponents/styling/_tokens.scss | 8 + src/plugins/AIComponents/styling/index.scss | 1 + .../styling/voice-visualization.scss | 38 ++ 53 files changed, 3364 insertions(+), 77 deletions(-) create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceMode.scss create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceMode.tsx create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx create mode 100644 examples/ai-chatbot/src/components/VoiceMode/index.ts create mode 100644 examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts create mode 100644 examples/ai-chatbot/src/components/useSendAIMessage.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceWave.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/bounce.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/driver.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/index.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/math.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/palette.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/renderers/glowWebGLRenderer.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/renderers/types.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/renderers/waveRenderer.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/types.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/useAudioLevel.ts create mode 100644 src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx create mode 100644 src/plugins/AIComponents/__tests__/useAudioLevel.test.ts create mode 100644 src/plugins/AIComponents/__tests__/voiceDriver.test.ts create mode 100644 src/plugins/AIComponents/__tests__/voicePalette.test.ts create mode 100644 src/plugins/AIComponents/styling/voice-visualization.scss diff --git a/examples/ai-chatbot/README.md b/examples/ai-chatbot/README.md index c5a24c4f7..8956514da 100644 --- a/examples/ai-chatbot/README.md +++ b/examples/ai-chatbot/README.md @@ -1,7 +1,7 @@ # AI chatbot example A Next.js chatbot built with `stream-chat-react` and its `stream-chat-react/ai-components` entry point -(streaming messages, AI state indicator, AI message composer). Styles come from +(streaming messages, AI state indicator, AI message composer, voice agent visualizations). Styles come from `stream-chat-react/css/ai-components.css`. ## Getting started @@ -27,4 +27,10 @@ The hosted demo server only answers for channels of Stream's demo app. With your `STREAM_API_SECRET` the AI agent never joins your channels, so run your own AI agent server (it must expose `POST /start-ai-agent` and `POST /summarize`, as called from `src/components/api.ts`) and point `NEXT_PUBLIC_AI_SERVER_URL` at it. +## Voice mode + +The waveform button in the composer opens voice mode: a hands-free conversation in which the AI agent is shown as a `VoiceGlow` and your microphone as a `VoiceWave`. It listens to you, sends what you said, and reads the answer aloud before listening again. + +It needs a browser with the Web Speech API's `SpeechRecognition` (Chrome, Edge or Safari; the button is hidden elsewhere) and access to the microphone. Reading answers aloud uses `speechSynthesis`; where it is missing, the answer is only shown in the chat. + See [implementation.md](./implementation.md) for implementation notes. diff --git a/examples/ai-chatbot/implementation.md b/examples/ai-chatbot/implementation.md index c449dbab3..163b21bdb 100644 --- a/examples/ai-chatbot/implementation.md +++ b/examples/ai-chatbot/implementation.md @@ -340,6 +340,23 @@ examples/ai-chatbot/ - Posts to `/summarize` endpoint with OpenAI platform - Returns summary string from API response +### 18. **useSendAIMessage.ts** + +- The send path shared by `MessageInputBar` and voice mode: composes the message (with attachments), watches the channel if needed, starts the AI agent when it is not watching yet, sends, records the message against the rate limit and summarizes a new conversation +- Also holds `availableModels`, `defaultPlatformModel` and `isWatchedByAI` + +### 19. **VoiceMode** + +A hands-free conversation with the agent, built on `VoiceGlow`, `VoiceWave`, `useAudioLevel` and `useSpeechToText` from `stream-chat-react/ai-components`. + +- `VoiceModeButton` sits in the composer next to the dictation button and appears only where the browser has `SpeechRecognition` (Chrome, Edge, Safari) +- `VoiceModeProvider` (in `ChatContainer`) holds whether voice mode is open; `VoiceMode` is mounted inside `Channel` only while open, so the microphone is released when it closes +- The overlay shows a large `VoiceGlow` for the agent and a `VoiceWave` for the user's microphone, fed by `useAudioLevel` on the `getUserMedia` stream +- `useVoiceConversation` runs the turn loop: listen (dictation ends on silence), send through `useSendAIMessage`, wait for the agent's `ai_indicator` events, read the finished answer aloud with `speechSynthesis` (markdown stripped, code skipped), then listen again +- Voice agent state: `connecting` while the microphone is requested or the agent is being started, `listening` during the user's turn, `thinking` while the agent works, `speaking` while the answer is read aloud, `idle` when paused or stopped +- Pause/resume and end buttons; Escape closes it; the rate limit and a denied microphone stop it with a message +- `--str-chat__ai-voice-backdrop` is set to the demo background in `index.scss`, so the visualizations pick their light or dark look from the demo theme rather than the SDK theme + --- ## Next.js Architecture diff --git a/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx b/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx index 9ffa6bc1b..5e9027ef4 100644 --- a/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx +++ b/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx @@ -15,6 +15,7 @@ import { MessageBubble } from '../MessageBubble'; import { MessageInputBar } from '../MessageInputBar'; import { AIStateIndicator } from '../AIStateIndicator'; import { TopNavBar } from '../TopNavBar'; +import { VoiceMode, VoiceModeProvider } from '../VoiceMode'; import './ChatContainer.scss'; interface ChatContainerProps { @@ -50,14 +51,17 @@ export const ChatContainer = ({ onToggleSidebar }: ChatContainerProps) => { MessageComposerUI: MessageInputBar, }} > - - - - - - - - + + + + + + + + + + + ); diff --git a/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx b/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx index 3880f9988..f2cccf77f 100644 --- a/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx +++ b/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx @@ -3,44 +3,26 @@ import { AIMessageComposer } from 'stream-chat-react/ai-components'; import { useEffect, useState } from 'react'; import { - type Channel, isImageFile, type LocalUploadAttachment, type UploadRequestFn, } from 'stream-chat'; import { useAttachmentsForPreview, - useChannelActionContext, useChannelStateContext, useChatContext, useMessageComposerController, } from 'stream-chat-react'; -import { startAiAgent, summarizeConversation } from '@/components/api'; -import { - checkRateLimit, - formatTimeRemaining, - recordMessage, -} from '@/components/rateLimitUtils'; +import { checkRateLimit, formatTimeRemaining } from '@/components/rateLimitUtils'; +import { availableModels, useSendAIMessage } from '@/components/useSendAIMessage'; +import { VoiceModeButton } from '@/components/VoiceMode'; import './MessageInputBar.scss'; -const isWatchedByAI = (channel: Channel) => - Object.keys(channel.state.watchers).some((watcher) => watcher.startsWith('ai-bot')); - -const availableModels = [ - { platform: 'openai', value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, - { platform: 'openai', value: 'gpt-5.4-nano', label: 'GPT-5.4 nano' }, - { - platform: 'gemini', - value: 'gemini-flash-latest', - label: 'Gemini Flash (latest)', - }, -] as const; - export const MessageInputBar = () => { const { client } = useChatContext(); - const { updateMessage, sendMessage } = useChannelActionContext(); const { channel } = useChannelStateContext(); const composer = useMessageComposerController(); + const sendAIMessage = useSendAIMessage(); const { attachments } = useAttachmentsForPreview(); const [selectedPlatformModel, setSelectedPlatformModel] = useState(); @@ -124,46 +106,14 @@ export const MessageInputBar = () => { const platformModel = formData.get('platform-model'); setSelectedPlatformModel(platformModel as string); - composer.textComposer.setText(message as string); - - const composedData = await composer.compose(); - - if (!composedData) return; - - target.reset(); - composer.clear(); - - updateMessage(composedData?.localMessage); - - if (!channel.initialized) { - await channel.watch(); - } - - const [platform, model] = (platformModel as string).split('|'); - - if (!isWatchedByAI(channel)) { - await startAiAgent(channel, model, platform); - } - - await sendMessage(composedData); - - // Record message after successful send - recordMessage(channel.id!); + const sent = await sendAIMessage({ + onComposed: () => target.reset(), + platformModel: platformModel as string, + text: message as string, + }); // Update rate limit state - const newState = checkRateLimit(channel.id!); - setRateLimitState(newState); - - if (typeof channel.data?.summary !== 'string' || !channel.data.summary.length) { - const summary = await summarizeConversation(message as string).catch(() => { - console.warn('Failed to summarize conversation'); - return null; - }); - - if (typeof summary === 'string' && summary.length > 0) { - await channel.update({ summary }); - } - } + if (sent) setRateLimitState(checkRateLimit(channel.id!)); }} > @@ -200,6 +150,7 @@ export const MessageInputBar = () => {
+ = { + error: 'Voice mode stopped', + listening: 'Listening', + paused: 'Paused', + speaking: 'Speaking', + starting: 'Connecting', + waiting: 'Thinking', +}; + +const VoiceConversation = () => { + const { close } = useVoiceMode(); + const { agentState, error, micLevel, pause, phase, reply, resume, transcript } = + useVoiceConversation(); + const closeRef = useRef(null); + + useEffect(() => { + closeRef.current?.focus(); + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') close(); + }; + document.addEventListener('keydown', onKeyDown); + return () => document.removeEventListener('keydown', onKeyDown); + }, [close]); + + const paused = phase === 'paused'; + const caption = + phase === 'error' + ? error + : phase === 'speaking' + ? reply + : phase === 'listening' + ? transcript || 'Go ahead, I’m listening.' + : phase === 'waiting' + ? transcript + : ''; + + return ( +
+
+ +

+ {STATUS[phase]} +

+

{caption}

+
+ + {/* The glow is the agent; the wave under it is the user's microphone. */} + + +
+ + +
+
+ ); +}; + +/** + * A hands-free conversation with the agent over the chat: it listens, sends what was said, + * shows the agent thinking and reads its answer aloud. Mounted only while open, so the + * microphone is released as soon as it closes. + */ +export const VoiceMode = () => { + const { isOpen } = useVoiceMode(); + return isOpen ? : null; +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx new file mode 100644 index 000000000..c28a69f93 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx @@ -0,0 +1,35 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useVoiceMode } from './VoiceModeContext'; + +const supportsVoiceMode = () => + typeof window !== 'undefined' && + !!( + (window as Window & { SpeechRecognition?: unknown }).SpeechRecognition ?? + (window as Window & { webkitSpeechRecognition?: unknown }).webkitSpeechRecognition + ) && + !!navigator.mediaDevices?.getUserMedia; + +/** Opens voice mode; shown only where the browser can listen (Chrome, Edge, Safari). */ +export const VoiceModeButton = () => { + const { isOpen, open } = useVoiceMode(); + // Support is known only in the browser, so the button appears after hydration. + const [isSupported, setIsSupported] = useState(false); + useEffect(() => setIsSupported(supportsVoiceMode()), []); + + if (!isSupported) return null; + + return ( + + ); +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx new file mode 100644 index 000000000..9018bc8a8 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx @@ -0,0 +1,34 @@ +'use client'; + +import { createContext, type ReactNode, useContext, useMemo, useState } from 'react'; + +type VoiceModeContextValue = { + close: () => void; + isOpen: boolean; + open: () => void; +}; + +const VoiceModeContext = createContext(undefined); + +export const useVoiceMode = () => { + const context = useContext(VoiceModeContext); + if (!context) { + throw new Error('useVoiceMode must be used within VoiceModeProvider'); + } + return context; +}; + +export const VoiceModeProvider = ({ children }: { children: ReactNode }) => { + const [isOpen, setIsOpen] = useState(false); + + const value = useMemo( + () => ({ + close: () => setIsOpen(false), + isOpen, + open: () => setIsOpen(true), + }), + [isOpen], + ); + + return {children}; +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/index.ts b/examples/ai-chatbot/src/components/VoiceMode/index.ts new file mode 100644 index 000000000..cfe736dc0 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/index.ts @@ -0,0 +1,3 @@ +export * from './VoiceMode'; +export * from './VoiceModeButton'; +export * from './VoiceModeContext'; diff --git a/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts b/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts new file mode 100644 index 000000000..e1108d880 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts @@ -0,0 +1,259 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; +import type { Channel } from 'stream-chat'; +import { + AIStates, + useAIState, + useChannelStateContext, + useChatContext, +} from 'stream-chat-react'; +import { + useAudioLevel, + useSpeechToText, + type VoiceAgentState, +} from 'stream-chat-react/ai-components'; +import { checkRateLimit } from '@/components/rateLimitUtils'; +import { isWatchedByAI, useSendAIMessage } from '@/components/useSendAIMessage'; + +/** + * Where the conversation is: + * - `starting`: asking for the microphone; + * - `listening`: dictating the user's turn; + * - `waiting`: the turn was sent and the agent is working on its answer; + * - `speaking`: reading the answer aloud; + * - `paused`: the user muted voice mode; + * - `error`: it can't go on (no microphone, rate limit, ...). + */ +export type VoicePhase = + | 'starting' + | 'listening' + | 'waiting' + | 'speaking' + | 'paused' + | 'error'; + +const WORKING_STATES: string[] = [ + AIStates.Thinking, + AIStates.ExternalSources, + AIStates.Generating, +]; + +/** Long answers are cut, as browsers stop reading aloud after a while. */ +const MAX_SPOKEN_LENGTH = 1200; + +const canSpeak = () => typeof window !== 'undefined' && 'speechSynthesis' in window; + +/** Markdown as plain prose for reading aloud: code is skipped, formatting dropped. */ +export const toSpeech = (markdown: string) => + markdown + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`([^`]*)`/g, '$1') + .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/^\s{0,3}(#{1,6}|>|[-*+]|\d+\.)\s+/gm, '') + .replace(/[*_~|]/g, '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, MAX_SPOKEN_LENGTH); + +const lastAgentMessageText = (channel: Channel, userId?: string) => { + const messages = channel.state.messages; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].user?.id !== userId) return messages[i].text ?? ''; + } + return ''; +}; + +/** + * A hands-free conversation with the AI agent on the current channel: listen to the user, + * send what they said, wait for the agent's answer, read it aloud, then listen again. + */ +export const useVoiceConversation = () => { + const { client } = useChatContext(); + const { channel } = useChannelStateContext(); + const { aiState } = useAIState(channel); + const sendAIMessage = useSendAIMessage(); + + const [phase, setPhase] = useState('starting'); + const [mic, setMic] = useState(null); + const [transcript, setTranscript] = useState(''); + const [reply, setReply] = useState(''); + const [error, setError] = useState(null); + const [agentStarting, setAgentStarting] = useState(false); + + const phaseRef = useRef(phase); + useEffect(() => { + phaseRef.current = phase; + }, [phase]); + const transcriptRef = useRef(''); + const sawAgentWorkRef = useRef(false); + const wasListeningRef = useRef(false); + + const micLevel = useAudioLevel(mic); + + const fail = useCallback((message: string) => { + setError(message); + setPhase('error'); + }, []); + + const { isListening, isSupported, startListening, stopListening } = useSpeechToText({ + onError: (message) => { + // Silence and aborts end a turn on their own; anything else stops voice mode. + if (/no speech|aborted/i.test(message)) return; + fail(message); + }, + onTranscript: (text) => { + transcriptRef.current = text; + setTranscript(text); + }, + }); + + // The microphone, for the level of the user's voice; released when voice mode closes. + useEffect(() => { + let stream: MediaStream | undefined; + let cancelled = false; + if (!navigator.mediaDevices?.getUserMedia) { + fail('This browser cannot use a microphone.'); + return; + } + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then((granted) => { + if (cancelled) { + granted.getTracks().forEach((track) => track.stop()); + return; + } + stream = granted; + setMic(granted); + setPhase((current) => (current === 'starting' ? 'listening' : current)); + }) + .catch(() => fail('Voice mode needs access to your microphone.')); + return () => { + cancelled = true; + stream?.getTracks().forEach((track) => track.stop()); + }; + }, [fail]); + + // Start dictating whenever it is the user's turn and recognition is not running. + useEffect(() => { + if (phase !== 'listening' || isListening || !isSupported) return; + // Recognition can refuse to restart in the same tick it ended. + const timeout = setTimeout(() => { + try { + transcriptRef.current = ''; + setTranscript(''); + startListening(); + } catch { + // Already started; the next end restarts it. + } + }, 150); + return () => clearTimeout(timeout); + }, [isListening, isSupported, phase, startListening]); + + const submit = useCallback( + (text: string) => { + if (checkRateLimit(channel.id!).isLimited) { + fail('Limit reached, 10 messages per conversation.'); + return; + } + sawAgentWorkRef.current = false; + setReply(''); + setAgentStarting(!isWatchedByAI(channel)); + setPhase('waiting'); + sendAIMessage({ text }).catch(() => fail('Your message could not be sent.')); + }, + [channel, fail, sendAIMessage], + ); + + // A turn ends when recognition stops: send what was said, or listen again after silence. + useEffect(() => { + if (isListening) { + wasListeningRef.current = true; + return; + } + if (!wasListeningRef.current) return; + wasListeningRef.current = false; + if (phaseRef.current !== 'listening') return; + const text = transcriptRef.current.trim(); + if (text) submit(text); + }, [isListening, submit]); + + const speak = useCallback((markdown: string) => { + const text = toSpeech(markdown); + if (!text || !canSpeak()) { + setPhase('listening'); + return; + } + setReply(text); + setPhase('speaking'); + const utterance = new SpeechSynthesisUtterance(text); + const done = () => { + if (phaseRef.current === 'speaking') setPhase('listening'); + }; + utterance.onend = done; + utterance.onerror = done; + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + }, []); + + // The agent's indicator: working while it thinks and writes, idle again once the answer is in. + useEffect(() => { + if (phaseRef.current !== 'waiting') return; + if (WORKING_STATES.includes(aiState)) { + sawAgentWorkRef.current = true; + setAgentStarting(false); + return; + } + if (!sawAgentWorkRef.current) return; + speak(lastAgentMessageText(channel, client.userID)); + }, [aiState, channel, client.userID, speak]); + + const pause = useCallback(() => { + setPhase('paused'); + stopListening(); + if (canSpeak()) window.speechSynthesis.cancel(); + }, [stopListening]); + + const resume = useCallback(() => setPhase('listening'), []); + + // Nothing keeps talking or listening once voice mode closes. + useEffect( + () => () => { + if (canSpeak()) window.speechSynthesis.cancel(); + }, + [], + ); + useEffect(() => { + if (phase === 'paused' || phase === 'error') stopListening(); + }, [phase, stopListening]); + + const agentState: VoiceAgentState = (() => { + switch (phase) { + case 'starting': + return 'connecting'; + case 'listening': + return 'listening'; + case 'waiting': + if (agentStarting) return 'connecting'; + // Without read-aloud, the answer being written is the agent speaking. + return aiState === AIStates.Generating && !canSpeak() ? 'speaking' : 'thinking'; + case 'speaking': + return 'speaking'; + default: + return 'idle'; + } + })(); + + return { + agentState, + error, + isSupported, + micLevel, + pause, + phase, + reply, + resume, + transcript, + }; +}; diff --git a/examples/ai-chatbot/src/components/index.scss b/examples/ai-chatbot/src/components/index.scss index 1cb4a39b0..578b1684a 100644 --- a/examples/ai-chatbot/src/components/index.scss +++ b/examples/ai-chatbot/src/components/index.scss @@ -96,6 +96,9 @@ --str-chat__ai-syntax-error: #f87171; --str-chat__ai-syntax-whitespace: hsla(0, 0%, 100%, 0.1); + /* VoiceWave and VoiceGlow sit on the demo's background, not the SDK theme's */ + --str-chat__ai-voice-backdrop: var(--ai-demo-bg-primary); + color-scheme: dark; color: var(--ai-demo-text-primary); background-color: var(--ai-demo-bg-primary); diff --git a/examples/ai-chatbot/src/components/useSendAIMessage.ts b/examples/ai-chatbot/src/components/useSendAIMessage.ts new file mode 100644 index 000000000..f18240112 --- /dev/null +++ b/examples/ai-chatbot/src/components/useSendAIMessage.ts @@ -0,0 +1,94 @@ +'use client'; + +import { useCallback } from 'react'; +import type { Channel } from 'stream-chat'; +import { + useChannelActionContext, + useChannelStateContext, + useMessageComposerController, +} from 'stream-chat-react'; +import { startAiAgent, summarizeConversation } from '@/components/api'; +import { recordMessage } from '@/components/rateLimitUtils'; + +export const availableModels = [ + { platform: 'openai', value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, + { platform: 'openai', value: 'gpt-5.4-nano', label: 'GPT-5.4 nano' }, + { + platform: 'gemini', + value: 'gemini-flash-latest', + label: 'Gemini Flash (latest)', + }, +] as const; + +export const defaultPlatformModel = `${availableModels[0].platform}|${availableModels[0].value}`; + +export const isWatchedByAI = (channel: Channel) => + Object.keys(channel.state.watchers).some((watcher) => watcher.startsWith('ai-bot')); + +type SendAIMessageOptions = { + text: string; + /** `platform|model`, as the model select submits it. */ + platformModel?: string; + /** Called once the message is composed, before the agent is started and it is sent. */ + onComposed?: () => void; +}; + +/** + * Sends a message to the AI agent: composes it (with any attachments), starts the agent on the + * channel if it isn't watching yet, sends it, records it against the rate limit and gives a new + * conversation its summary. Shared by the composer and voice mode. + * + * Resolves to whether a message was sent. + */ +export const useSendAIMessage = () => { + const { updateMessage, sendMessage } = useChannelActionContext(); + const { channel } = useChannelStateContext(); + const composer = useMessageComposerController(); + + return useCallback( + async ({ + text, + platformModel = defaultPlatformModel, + onComposed, + }: SendAIMessageOptions) => { + composer.textComposer.setText(text); + + const composedData = await composer.compose(); + + if (!composedData) return false; + + onComposed?.(); + composer.clear(); + + updateMessage(composedData.localMessage); + + if (!channel.initialized) { + await channel.watch(); + } + + const [platform, model] = platformModel.split('|'); + + if (!isWatchedByAI(channel)) { + await startAiAgent(channel, model, platform); + } + + await sendMessage(composedData); + + recordMessage(channel.id!); + + if (typeof channel.data?.summary !== 'string' || !channel.data.summary.length) { + const summary = await summarizeConversation(text).catch(() => { + console.warn('Failed to summarize conversation'); + return null; + }); + + if (typeof summary === 'string' && summary.length > 0) { + await channel.update({ summary }); + } + } + + return true; + }, + [channel, composer, sendMessage, updateMessage], + ); +}; diff --git a/src/i18n/de.json b/src/i18n/de.json index dfee28918..449bc36c8 100644 --- a/src/i18n/de.json +++ b/src/i18n/de.json @@ -226,6 +226,7 @@ "aria/Unpin Message": "Anheftung aufheben", "aria/User selected: {{ user }}": "Benutzer ausgewählt: {{ user }}", "aria/video": "Video", + "aria/Voice agent: {{ state }}": "Sprachagent: {{ state }}", "aria/voice message": "Sprachnachricht", "aria/Voice message sent": "Sprachnachricht gesendet", "aria/Voice recording attached": "Sprachaufnahme angehängt", @@ -267,6 +268,7 @@ "Command not available while replying": "Befehl beim Antworten nicht verfügbar", "Commands": "Befehle", "Commands matching": "Übereinstimmende Befehle", + "Connecting": "Verbinde", "Connecting the dots": "Ich verbinde die Punkte", "Connection failure, reconnecting now...": "Verbindungsfehler, Wiederherstellung der Verbindung...", "Consulting the AI gods": "Ich befrage die KI-Götter", @@ -445,6 +447,7 @@ "Link": "Link", "linkCount_one": "Link", "linkCount_other": "{{ count }} Links", + "Listening": "Hört zu", "live": "live", "Live for {{duration}}": "Live für {{duration}}", "Live location": "Live-Standort", @@ -545,6 +548,7 @@ "Quote Reply": "Zitat-Antwort", "Reached the vote limit. Remove an existing vote first.": "Das Abstimmungslimit wurde erreicht. Entfernen Sie zuerst eine bestehende Stimme.", "Reading the digital tea leaves": "Ich lese die digitalen Teeblätter", + "Ready": "Bereit", "Recording format is not supported and cannot be reproduced": "Aufnahmeformat wird nicht unterstützt und kann nicht wiedergegeben werden", "Remind me": "Erinnern", "Remind Me": "Erinnern", @@ -613,6 +617,7 @@ "Slow Mode ON": "Langsamer Modus EIN", "Slow mode, wait {{ seconds }}s...": "Langsamer Modus, warte {{ seconds }}s...", "Some of the files will not be accepted": "Einige der Dateien werden nicht akzeptiert", + "Speaking": "Spricht", "Start typing to search": "Tippen Sie, um zu suchen", "Stop sharing": "Teilen beenden", "Submit": "Absenden", @@ -621,6 +626,7 @@ "Summoning my inner genius": "Ich beschwöre mein inneres Genie", "Tap to remove": "Tippen zum Entfernen", "Tap to remove: {{ reactionName }}": "Tippen zum Entfernen: {{ reactionName }}", + "Thinking": "Denkt nach", "Thinking really hard": "Ich denke angestrengt nach", "Thinking...": "Denken...", "this content could not be displayed": "Dieser Inhalt konnte nicht angezeigt werden", diff --git a/src/i18n/en.json b/src/i18n/en.json index 5adefcc7b..a9d61b90b 100644 --- a/src/i18n/en.json +++ b/src/i18n/en.json @@ -226,6 +226,7 @@ "aria/Unpin Message": "Unpin Message", "aria/User selected: {{ user }}": "User selected: {{ user }}", "aria/video": "video", + "aria/Voice agent: {{ state }}": "Voice agent: {{ state }}", "aria/voice message": "voice message", "aria/Voice message sent": "Voice message sent", "aria/Voice recording attached": "Voice recording attached", @@ -267,6 +268,7 @@ "Command not available while replying": "Command not available while replying", "Commands": "Commands", "Commands matching": "Commands matching", + "Connecting": "Connecting", "Connecting the dots": "Connecting the dots", "Connection failure, reconnecting now...": "Connection failure, reconnecting now...", "Consulting the AI gods": "Consulting the AI gods", @@ -445,6 +447,7 @@ "Link": "Link", "linkCount_one": "Link", "linkCount_other": "{{ count }} links", + "Listening": "Listening", "live": "live", "Live for {{duration}}": "Live for {{duration}}", "Live location": "Live location", @@ -545,6 +548,7 @@ "Quote Reply": "Quote Reply", "Reached the vote limit. Remove an existing vote first.": "Reached the vote limit. Remove an existing vote first.", "Reading the digital tea leaves": "Reading the digital tea leaves", + "Ready": "Ready", "Recording format is not supported and cannot be reproduced": "Recording format is not supported and cannot be reproduced", "Remind me": "Remind me", "Remind Me": "Remind Me", @@ -613,6 +617,7 @@ "Slow Mode ON": "Slow Mode ON", "Slow mode, wait {{ seconds }}s...": "Slow mode, wait {{ seconds }}s...", "Some of the files will not be accepted": "Some of the files will not be accepted", + "Speaking": "Speaking", "Start typing to search": "Start typing to search", "Stop sharing": "Stop sharing", "Submit": "Submit", @@ -621,6 +626,7 @@ "Summoning my inner genius": "Summoning my inner genius", "Tap to remove": "Tap to remove", "Tap to remove: {{ reactionName }}": "Tap to remove: {{ reactionName }}", + "Thinking": "Thinking", "Thinking really hard": "Thinking really hard", "Thinking...": "Thinking...", "this content could not be displayed": "this content could not be displayed", diff --git a/src/i18n/es.json b/src/i18n/es.json index 773b86087..aff84ebbb 100644 --- a/src/i18n/es.json +++ b/src/i18n/es.json @@ -243,6 +243,7 @@ "aria/Unpin Message": "Desfijar mensaje", "aria/User selected: {{ user }}": "Usuario seleccionado: {{ user }}", "aria/video": "vídeo", + "aria/Voice agent: {{ state }}": "Agente de voz: {{ state }}", "aria/voice message": "mensaje de voz", "aria/Voice message sent": "Mensaje de voz enviado", "aria/Voice recording attached": "Grabación de voz adjuntada", @@ -284,6 +285,7 @@ "Command not available while replying": "Comando no disponible mientras se responde", "Commands": "Comandos", "Commands matching": "Coincidencia de comandos", + "Connecting": "Conectando", "Connecting the dots": "Conectando los puntos", "Connection failure, reconnecting now...": "Fallo de conexión, reconectando ahora...", "Consulting the AI gods": "Consultando a los dioses de la IA", @@ -465,6 +467,7 @@ "linkCount_one": "Enlace", "linkCount_many": "{{ count }} enlaces", "linkCount_other": "{{ count }} enlaces", + "Listening": "Escuchando", "live": "En vivo", "Live for {{duration}}": "En vivo durante {{duration}}", "Live location": "Ubicación en vivo", @@ -565,6 +568,7 @@ "Quote Reply": "Responder con cita", "Reached the vote limit. Remove an existing vote first.": "Se ha alcanzado el límite de votos. Elimina un voto existente primero.", "Reading the digital tea leaves": "Leyendo las hojas de té digitales", + "Ready": "Listo", "Recording format is not supported and cannot be reproduced": "El formato de grabación no es compatible y no se puede reproducir", "Remind me": "Recordarme", "Remind Me": "Recordarme", @@ -638,6 +642,7 @@ "Slow Mode ON": "Modo lento activado", "Slow mode, wait {{ seconds }}s...": "Modo lento, espera {{ seconds }} s...", "Some of the files will not be accepted": "Algunos archivos no serán aceptados", + "Speaking": "Hablando", "Start typing to search": "Empieza a escribir para buscar", "Stop sharing": "Dejar de compartir", "Submit": "Enviar", @@ -646,6 +651,7 @@ "Summoning my inner genius": "Invocando a mi genio interior", "Tap to remove": "Toca para quitar", "Tap to remove: {{ reactionName }}": "Toca para quitar: {{ reactionName }}", + "Thinking": "Pensando", "Thinking really hard": "Pensando muy fuerte", "Thinking...": "Pensando...", "this content could not be displayed": "Este contenido no se pudo mostrar", diff --git a/src/i18n/fr.json b/src/i18n/fr.json index 3c13008d0..d5120dfd3 100644 --- a/src/i18n/fr.json +++ b/src/i18n/fr.json @@ -243,6 +243,7 @@ "aria/Unpin Message": "Détacher le message", "aria/User selected: {{ user }}": "Utilisateur sélectionné : {{ user }}", "aria/video": "vidéo", + "aria/Voice agent: {{ state }}": "Agent vocal : {{ state }}", "aria/voice message": "message vocal", "aria/Voice message sent": "Message vocal envoyé", "aria/Voice recording attached": "Enregistrement vocal joint", @@ -284,6 +285,7 @@ "Command not available while replying": "Commande non disponible pendant la réponse", "Commands": "Commandes", "Commands matching": "Correspondance des commandes", + "Connecting": "Connexion", "Connecting the dots": "Je relie les points", "Connection failure, reconnecting now...": "Échec de la connexion, reconnexion en cours...", "Consulting the AI gods": "Je consulte les dieux de l'IA", @@ -465,6 +467,7 @@ "linkCount_one": "Lien", "linkCount_many": "{{ count }} liens", "linkCount_other": "{{ count }} liens", + "Listening": "À l’écoute", "live": "en direct", "Live for {{duration}}": "En direct pendant {{duration}}", "Live location": "Emplacement en direct", @@ -565,6 +568,7 @@ "Quote Reply": "Répondre par citation", "Reached the vote limit. Remove an existing vote first.": "La limite de votes a été atteinte. Supprimez d'abord un vote existant.", "Reading the digital tea leaves": "Je lis dans le marc de café numérique", + "Ready": "Prêt", "Recording format is not supported and cannot be reproduced": "Le format d'enregistrement n'est pas pris en charge et ne peut pas être reproduit", "Remind me": "Me rappeler", "Remind Me": "Me rappeler", @@ -638,6 +642,7 @@ "Slow Mode ON": "Mode lent activé", "Slow mode, wait {{ seconds }}s...": "Mode lent, attendez {{ seconds }} s...", "Some of the files will not be accepted": "Certains fichiers ne seront pas acceptés", + "Speaking": "Parle", "Start typing to search": "Commencez à taper pour rechercher", "Stop sharing": "Arrêter de partager", "Submit": "Envoyer", @@ -646,6 +651,7 @@ "Summoning my inner genius": "J'invoque mon génie intérieur", "Tap to remove": "Appuyez pour retirer", "Tap to remove: {{ reactionName }}": "Appuyez pour retirer: {{ reactionName }}", + "Thinking": "Réfléchit", "Thinking really hard": "Je réfléchis très fort", "Thinking...": "Réflexion...", "this content could not be displayed": "ce contenu n'a pas pu être affiché", diff --git a/src/i18n/hi.json b/src/i18n/hi.json index 515986503..9ec9fa6dc 100644 --- a/src/i18n/hi.json +++ b/src/i18n/hi.json @@ -226,6 +226,7 @@ "aria/Unpin Message": "संदेश अनपिन करें", "aria/User selected: {{ user }}": "उपयोगकर्ता चयनित: {{ user }}", "aria/video": "वीडियो", + "aria/Voice agent: {{ state }}": "वॉइस एजेंट: {{ state }}", "aria/voice message": "वॉइस मैसेज", "aria/Voice message sent": "वॉइस संदेश भेजा गया", "aria/Voice recording attached": "वॉइस रिकॉर्डिंग संलग्न की गई", @@ -267,6 +268,7 @@ "Command not available while replying": "उत्तर देते समय कमांड उपलब्ध नहीं है", "Commands": "कमांड", "Commands matching": "मेल खाती है", + "Connecting": "कनेक्ट हो रहा है", "Connecting the dots": "कड़ियाँ जोड़ रहा हूँ", "Connection failure, reconnecting now...": "कनेक्शन विफल रहा, अब पुनः कनेक्ट हो रहा है ...", "Consulting the AI gods": "एआई देवताओं से सलाह ले रहा हूँ", @@ -446,6 +448,7 @@ "Link": "लिंक", "linkCount_one": "1 लिंक", "linkCount_other": "{{ count }} लिंक", + "Listening": "सुन रहा है", "live": "लाइव", "Live for {{duration}}": "{{duration}} के लिए लाइव", "Live location": "लाइव स्थान", @@ -546,6 +549,7 @@ "Quote Reply": "उद्धरण जवाब", "Reached the vote limit. Remove an existing vote first.": "मतदान सीमा तक पहुंच गया। पहले एक मौजूदा वोट हटाएं।", "Reading the digital tea leaves": "डिजिटल चाय की पत्तियाँ पढ़ रहा हूँ", + "Ready": "तैयार", "Recording format is not supported and cannot be reproduced": "रेकॉर्डिंग फ़ॉर्मेट समर्थित नहीं है और पुनः उत्पन्न नहीं किया जा सकता", "Remind me": "मुझे याद दिलाएं", "Remind Me": "मुझे याद दिलाएं", @@ -614,6 +618,7 @@ "Slow Mode ON": "स्लो मोड ऑन", "Slow mode, wait {{ seconds }}s...": "स्लो मोड, {{ seconds }} सेकंड प्रतीक्षा करें...", "Some of the files will not be accepted": "कुछ फ़ाइलें स्वीकार नहीं की जाएंगी", + "Speaking": "बोल रहा है", "Start typing to search": "खोजने के लिए टाइप करना शुरू करें", "Stop sharing": "साझा करना बंद करें", "Submit": "जमा करें", @@ -622,6 +627,7 @@ "Summoning my inner genius": "अपने अंदर की प्रतिभा को बुला रहा हूँ", "Tap to remove": "हटाने के लिए टैप करें", "Tap to remove: {{ reactionName }}": "हटाने के लिए टैप करें: {{ reactionName }}", + "Thinking": "सोच रहा है", "Thinking really hard": "बहुत गहराई से सोच रहा हूँ", "Thinking...": "सोच रहा है...", "this content could not be displayed": "यह कॉन्टेंट लोड नहीं हो पाया", diff --git a/src/i18n/it.json b/src/i18n/it.json index 1ec723099..b170b818e 100644 --- a/src/i18n/it.json +++ b/src/i18n/it.json @@ -243,6 +243,7 @@ "aria/Unpin Message": "Rimuovi messaggio appuntato", "aria/User selected: {{ user }}": "Utente selezionato: {{ user }}", "aria/video": "video", + "aria/Voice agent: {{ state }}": "Agente vocale: {{ state }}", "aria/voice message": "messaggio vocale", "aria/Voice message sent": "Messaggio vocale inviato", "aria/Voice recording attached": "Registrazione vocale allegata", @@ -284,6 +285,7 @@ "Command not available while replying": "Comando non disponibile durante la risposta", "Commands": "Comandi", "Commands matching": "Comandi corrispondenti", + "Connecting": "Connessione", "Connecting the dots": "Collego i puntini", "Connection failure, reconnecting now...": "Errore di connessione, riconnessione in corso...", "Consulting the AI gods": "Consulto gli dei dell'IA", @@ -465,6 +467,7 @@ "linkCount_one": "Link", "linkCount_many": "{{ count }} link", "linkCount_other": "{{ count }} link", + "Listening": "In ascolto", "live": "live", "Live for {{duration}}": "Live per {{duration}}", "Live location": "Posizione live", @@ -565,6 +568,7 @@ "Quote Reply": "Rispondi con citazione", "Reached the vote limit. Remove an existing vote first.": "Raggiunto il limite di voti. Rimuovi prima un voto esistente.", "Reading the digital tea leaves": "Leggo i fondi di tè digitali", + "Ready": "Pronto", "Recording format is not supported and cannot be reproduced": "Il formato di registrazione non è supportato e non può essere riprodotto", "Remind me": "Promemoria", "Remind Me": "Ricordami", @@ -638,6 +642,7 @@ "Slow Mode ON": "Modalità lenta attivata", "Slow mode, wait {{ seconds }}s...": "Modalità lenta, attendi {{ seconds }} s...", "Some of the files will not be accepted": "Alcuni dei file non saranno accettati", + "Speaking": "Sta parlando", "Start typing to search": "Inizia a digitare per cercare", "Stop sharing": "Ferma condivisione", "Submit": "Invia", @@ -646,6 +651,7 @@ "Summoning my inner genius": "Evoco il mio genio interiore", "Tap to remove": "Tocca per rimuovere", "Tap to remove: {{ reactionName }}": "Tocca per rimuovere: {{ reactionName }}", + "Thinking": "Sta pensando", "Thinking really hard": "Sto pensando intensamente", "Thinking...": "Pensando...", "this content could not be displayed": "questo contenuto non può essere mostrato", diff --git a/src/i18n/ja.json b/src/i18n/ja.json index fcac44336..fe6db9701 100644 --- a/src/i18n/ja.json +++ b/src/i18n/ja.json @@ -217,6 +217,7 @@ "aria/Unpin Message": "ピンを解除", "aria/User selected: {{ user }}": "選択したユーザー：{{ user }}", "aria/video": "動画", + "aria/Voice agent: {{ state }}": "音声エージェント: {{ state }}", "aria/voice message": "ボイスメッセージ", "aria/Voice message sent": "ボイスメッセージを送信しました", "aria/Voice recording attached": "ボイス録音を添付しました", @@ -258,6 +259,7 @@ "Command not available while replying": "返信中はコマンドを使用できません", "Commands": "コマンド", "Commands matching": "一致するコマンド", + "Connecting": "接続中", "Connecting the dots": "点と点をつなげています", "Connection failure, reconnecting now...": "接続が失敗しました。再接続中...", "Consulting the AI gods": "AIの神々に相談しています", @@ -433,6 +435,7 @@ "Limit votes per person": "1人あたりの投票数を制限する", "Link": "リンク", "linkCount_other": "{{ count }}件のリンク", + "Listening": "聞き取り中", "live": "ライブ", "Live for {{duration}}": "{{duration}}間ライブ", "Live location": "ライブ位置情報", @@ -533,6 +536,7 @@ "Quote Reply": "引用返信", "Reached the vote limit. Remove an existing vote first.": "投票制限に達しました。既存の投票を先に削除してください。", "Reading the digital tea leaves": "デジタルの茶葉を読み解いています", + "Ready": "準備完了", "Recording format is not supported and cannot be reproduced": "録音形式はサポートされておらず、再生できません", "Remind me": "リマインド", "Remind Me": "リマインダー", @@ -599,6 +603,7 @@ "Slow Mode ON": "スローモードオン", "Slow mode, wait {{ seconds }}s...": "スローモード、{{ seconds }}秒お待ちください...", "Some of the files will not be accepted": "一部のファイルは受け付けられません", + "Speaking": "話しています", "Start typing to search": "検索するには入力を開始してください", "Stop sharing": "共有を停止", "Submit": "送信", @@ -607,6 +612,7 @@ "Summoning my inner genius": "内なる天才を呼び覚ましています", "Tap to remove": "タップして削除", "Tap to remove: {{ reactionName }}": "タップして削除: {{ reactionName }}", + "Thinking": "考え中", "Thinking really hard": "一生懸命考えています", "Thinking...": "考え中...", "this content could not be displayed": "このコンテンツは表示できませんでした", diff --git a/src/i18n/ko.json b/src/i18n/ko.json index 59a3c12eb..57017eff1 100644 --- a/src/i18n/ko.json +++ b/src/i18n/ko.json @@ -217,6 +217,7 @@ "aria/Unpin Message": "핀 해제", "aria/User selected: {{ user }}": "선택한 사용자: {{ user }}", "aria/video": "동영상", + "aria/Voice agent: {{ state }}": "음성 에이전트: {{ state }}", "aria/voice message": "음성 메시지", "aria/Voice message sent": "음성 메시지 전송됨", "aria/Voice recording attached": "음성 녹음 첨부됨", @@ -258,6 +259,7 @@ "Command not available while replying": "답장 중에는 명령을 사용할 수 없습니다", "Commands": "명령어", "Commands matching": "일치하는 명령", + "Connecting": "연결 중", "Connecting the dots": "점들을 연결하는 중", "Connection failure, reconnecting now...": "연결 실패, 지금 다시 연결 중...", "Consulting the AI gods": "AI 신들께 여쭤보는 중", @@ -433,6 +435,7 @@ "Limit votes per person": "1인당 투표 수 제한", "Link": "링크", "linkCount_other": "링크 {{ count }}개", + "Listening": "듣는 중", "live": "라이브", "Live for {{duration}}": "{{duration}} 동안 라이브", "Live location": "라이브 위치", @@ -533,6 +536,7 @@ "Quote Reply": "인용 답장", "Reached the vote limit. Remove an existing vote first.": "투표 한도에 도달했습니다. 기존 투표를 먼저 제거하세요.", "Reading the digital tea leaves": "디지털 찻잎을 읽는 중", + "Ready": "준비됨", "Recording format is not supported and cannot be reproduced": "녹음 형식이 지원되지 않으므로 재생할 수 없습니다", "Remind me": "알림", "Remind Me": "알림 설정", @@ -599,6 +603,7 @@ "Slow Mode ON": "슬로우 모드 켜짐", "Slow mode, wait {{ seconds }}s...": "슬로우 모드, {{ seconds }}초 기다려 주세요...", "Some of the files will not be accepted": "일부 파일은 허용되지 않을 수 있습니다", + "Speaking": "말하는 중", "Start typing to search": "검색하려면 입력을 시작하세요", "Stop sharing": "공유 중지", "Submit": "제출", @@ -607,6 +612,7 @@ "Summoning my inner genius": "내면의 천재를 불러내는 중", "Tap to remove": "제거하려면 탭하세요", "Tap to remove: {{ reactionName }}": "제거하려면 탭하세요: {{ reactionName }}", + "Thinking": "생각 중", "Thinking really hard": "열심히 생각하는 중", "Thinking...": "생각 중...", "this content could not be displayed": "이 콘텐츠를 표시할 수 없습니다", diff --git a/src/i18n/nl.json b/src/i18n/nl.json index 317229391..e1db9c66b 100644 --- a/src/i18n/nl.json +++ b/src/i18n/nl.json @@ -226,6 +226,7 @@ "aria/Unpin Message": "Losmaken", "aria/User selected: {{ user }}": "Gebruiker geselecteerd: {{ user }}", "aria/video": "video", + "aria/Voice agent: {{ state }}": "Spraakassistent: {{ state }}", "aria/voice message": "spraakbericht", "aria/Voice message sent": "Spraakbericht verzonden", "aria/Voice recording attached": "Spraakopname bijgevoegd", @@ -267,6 +268,7 @@ "Command not available while replying": "Opdracht niet beschikbaar tijdens beantwoorden", "Commands": "Commando's", "Commands matching": "Bijpassende opdrachten", + "Connecting": "Verbinden", "Connecting the dots": "Ik verbind de puntjes", "Connection failure, reconnecting now...": "Verbindingsfout, opnieuw verbinden...", "Consulting the AI gods": "Ik raadpleeg de AI-goden", @@ -445,6 +447,7 @@ "Link": "Link", "linkCount_one": "Link", "linkCount_other": "{{ count }} links", + "Listening": "Luistert", "live": "live", "Live for {{duration}}": "Live voor {{duration}}", "Live location": "Live locatie", @@ -545,6 +548,7 @@ "Quote Reply": "Citaatantwoord", "Reached the vote limit. Remove an existing vote first.": "Stemlimiet bereikt. Verwijder eerst een bestaande stem.", "Reading the digital tea leaves": "Ik lees de digitale theeblaadjes", + "Ready": "Klaar", "Recording format is not supported and cannot be reproduced": "Opnameformaat wordt niet ondersteund en kan niet worden gereproduceerd", "Remind me": "Herinner me", "Remind Me": "Herinner mij", @@ -615,6 +619,7 @@ "Slow wait, wait {{ seconds }}s": "Langzame modus, wacht {{ seconds }}s", "Slow wait, wait {{ seconds }}s...": "Langzame modus, wacht {{ seconds }}s...", "Some of the files will not be accepted": "Sommige bestanden zullen niet worden geaccepteerd", + "Speaking": "Spreekt", "Start typing to search": "Begin met typen om te zoeken", "Stop sharing": "Delen stoppen", "Submit": "Versturen", @@ -623,6 +628,7 @@ "Summoning my inner genius": "Ik roep mijn innerlijke genie op", "Tap to remove": "Tik om te verwijderen", "Tap to remove: {{ reactionName }}": "Tik om te verwijderen: {{ reactionName }}", + "Thinking": "Denkt na", "Thinking really hard": "Ik denk heel diep na", "Thinking...": "Denken...", "this content could not be displayed": "Deze inhoud kan niet weergegeven worden", diff --git a/src/i18n/pt.json b/src/i18n/pt.json index 70fe0e6cd..b0e5c3192 100644 --- a/src/i18n/pt.json +++ b/src/i18n/pt.json @@ -243,6 +243,7 @@ "aria/Unpin Message": "Desfixar mensagem", "aria/User selected: {{ user }}": "Usuário selecionado: {{ user }}", "aria/video": "vídeo", + "aria/Voice agent: {{ state }}": "Agente de voz: {{ state }}", "aria/voice message": "mensagem de voz", "aria/Voice message sent": "Mensagem de voz enviada", "aria/Voice recording attached": "Gravação de voz anexada", @@ -284,6 +285,7 @@ "Command not available while replying": "Comando não disponível durante a resposta", "Commands": "Comandos", "Commands matching": "Comandos correspondentes", + "Connecting": "Conectando", "Connecting the dots": "Ligando os pontos", "Connection failure, reconnecting now...": "Falha de conexão, reconectando agora...", "Consulting the AI gods": "Consultando os deuses da IA", @@ -465,6 +467,7 @@ "linkCount_one": "Link", "linkCount_many": "{{ count }} links", "linkCount_other": "{{ count }} links", + "Listening": "Ouvindo", "live": "ao vivo", "Live for {{duration}}": "Ao vivo por {{duration}}", "Live location": "Localização ao vivo", @@ -565,6 +568,7 @@ "Quote Reply": "Responder com citação", "Reached the vote limit. Remove an existing vote first.": "Limite de votos atingido. Remova um voto existente primeiro.", "Reading the digital tea leaves": "Lendo as folhas de chá digitais", + "Ready": "Pronto", "Recording format is not supported and cannot be reproduced": "Formato de gravação não é suportado e não pode ser reproduzido", "Remind me": "Lembrar-me", "Remind Me": "Lembrar-me", @@ -638,6 +642,7 @@ "Slow Mode ON": "Modo lento LIGADO", "Slow mode, wait {{ seconds }}s...": "Modo lento, aguarde {{ seconds }} s...", "Some of the files will not be accepted": "Alguns arquivos não serão aceitos", + "Speaking": "Falando", "Start typing to search": "Comece a digitar para pesquisar", "Stop sharing": "Parar de compartilhar", "Submit": "Enviar", @@ -646,6 +651,7 @@ "Summoning my inner genius": "Invocando meu gênio interior", "Tap to remove": "Toque para remover", "Tap to remove: {{ reactionName }}": "Toque para remover: {{ reactionName }}", + "Thinking": "Pensando", "Thinking really hard": "Pensando bem forte", "Thinking...": "Pensando...", "this content could not be displayed": "este conteúdo não pôde ser exibido", diff --git a/src/i18n/ru.json b/src/i18n/ru.json index be09be9de..bdb9f115e 100644 --- a/src/i18n/ru.json +++ b/src/i18n/ru.json @@ -261,6 +261,7 @@ "aria/Unpin Message": "Открепить сообщение", "aria/User selected: {{ user }}": "Выбран пользователь: {{ user }}", "aria/video": "видео", + "aria/Voice agent: {{ state }}": "Голосовой агент: {{ state }}", "aria/voice message": "голосовое сообщение", "aria/Voice message sent": "Голосовое сообщение отправлено", "aria/Voice recording attached": "Голосовая запись прикреплена", @@ -302,6 +303,7 @@ "Command not available while replying": "Команда недоступна при ответе", "Commands": "Команды", "Commands matching": "Соответствие команд", + "Connecting": "Подключение", "Connecting the dots": "Соединяю точки", "Connection failure, reconnecting now...": "Ошибка соединения, переподключение...", "Consulting the AI gods": "Советуюсь с богами ИИ", @@ -489,6 +491,7 @@ "linkCount_few": "{{ count }} линка", "linkCount_many": "{{ count }} линков", "linkCount_other": "{{ count }} линков", + "Listening": "Слушает", "live": "В прямом эфире", "Live for {{duration}}": "В прямом эфире {{duration}}", "Live location": "Местоположение в прямом эфире", @@ -589,6 +592,7 @@ "Quote Reply": "Ответ со цитатой", "Reached the vote limit. Remove an existing vote first.": "Достигнут лимит голосов. Сначала удалите существующий голос.", "Reading the digital tea leaves": "Гадаю на цифровой чайной заварке", + "Ready": "Готов", "Recording format is not supported and cannot be reproduced": "Формат записи не поддерживается и не может быть воспроизведен", "Remind me": "Напомнить мне", "Remind Me": "Напомнить мне", @@ -667,6 +671,7 @@ "Slow Mode ON": "Медленный режим включен", "Slow mode, wait {{ seconds }}s...": "Медленный режим: подождите {{ seconds }} с...", "Some of the files will not be accepted": "Некоторые файлы не будут приняты", + "Speaking": "Говорит", "Start typing to search": "Начните вводить для поиска", "Stop sharing": "Прекратить делиться", "Submit": "Отправить", @@ -675,6 +680,7 @@ "Summoning my inner genius": "Пробуждаю внутреннего гения", "Tap to remove": "Нажмите, чтобы удалить", "Tap to remove: {{ reactionName }}": "Нажмите, чтобы удалить: {{ reactionName }}", + "Thinking": "Думает", "Thinking really hard": "Усердно думаю", "Thinking...": "Думаю...", "this content could not be displayed": "Этот контент не может быть отображен в данный момент", diff --git a/src/i18n/tr.json b/src/i18n/tr.json index a0ae5d703..c94716817 100644 --- a/src/i18n/tr.json +++ b/src/i18n/tr.json @@ -226,6 +226,7 @@ "aria/Unpin Message": "Sabitlemeyi kaldır", "aria/User selected: {{ user }}": "Seçilen kullanıcı: {{ user }}", "aria/video": "video", + "aria/Voice agent: {{ state }}": "Sesli asistan: {{ state }}", "aria/voice message": "sesli mesaj", "aria/Voice message sent": "Sesli mesaj gönderildi", "aria/Voice recording attached": "Ses kaydı eklendi", @@ -267,6 +268,7 @@ "Command not available while replying": "Yanıtlama sırasında komut kullanılamaz", "Commands": "Komutlar", "Commands matching": "Eşleşen komutlar", + "Connecting": "Bağlanıyor", "Connecting the dots": "Noktaları birleştiriyorum", "Connection failure, reconnecting now...": "Bağlantı hatası, tekrar bağlanılıyor...", "Consulting the AI gods": "Yapay zekâ tanrılarına danışıyorum", @@ -445,6 +447,7 @@ "Link": "Bağlantı", "linkCount_one": "Bağlantı", "linkCount_other": "{{ count }} bağlantı", + "Listening": "Dinliyor", "live": "canlı", "Live for {{duration}}": "{{duration}} boyunca canlı", "Live location": "Canlı konum", @@ -545,6 +548,7 @@ "Quote Reply": "Alıntıyla yanıtla", "Reached the vote limit. Remove an existing vote first.": "Oylama sınırına ulaşıldı. Önce mevcut bir oyu kaldırın.", "Reading the digital tea leaves": "Dijital çay yapraklarını okuyorum", + "Ready": "Hazır", "Recording format is not supported and cannot be reproduced": "Kayıt formatı desteklenmiyor ve çoğaltılamıyor", "Remind me": "Bana hatırlat", "Remind Me": "Hatırlat", @@ -613,6 +617,7 @@ "Slow Mode ON": "Yavaş Mod Açık", "Slow mode, wait {{ seconds }}s...": "Yavaş mod, {{ seconds }} sn bekleyin...", "Some of the files will not be accepted": "Bazı dosyalar kabul edilmeyecek", + "Speaking": "Konuşuyor", "Start typing to search": "Aramak için yazmaya başlayın", "Stop sharing": "Paylaşımı durdur", "Submit": "Gönder", @@ -621,6 +626,7 @@ "Summoning my inner genius": "İçimdeki dahiyi uyandırıyorum", "Tap to remove": "Kaldırmak için dokunun", "Tap to remove: {{ reactionName }}": "Kaldırmak için dokunun: {{ reactionName }}", + "Thinking": "Düşünüyor", "Thinking really hard": "Çok derin düşünüyorum", "Thinking...": "Düşünüyor...", "this content could not be displayed": "bu içerik gösterilemiyor", diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md index 75fe053b7..c283e426d 100644 --- a/src/plugins/AIComponents/README.md +++ b/src/plugins/AIComponents/README.md @@ -29,9 +29,9 @@ Load `ai-components.css` in addition to `stream-chat-react/css/index.css`. From `stream-chat-react/ai-components`: -- Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator` -- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText` -- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions` +- Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator`, `VoiceWave`, `VoiceGlow` +- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText`, `useAudioLevel` +- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions`, `VoiceAgentState`, `VoiceGlowRendering`, `VoiceVisualizationProps`, `VoiceWaveProps`, `VoiceGlowProps` `useMessageTextStreaming` is not exported from here; import it from the main entry point (`stream-chat-react`). @@ -190,6 +190,8 @@ Precedence for every token except the code palette is: your `--str-chat__ai-*` o | `table-text` | table cell text | `text-secondary` | `#495057` | | `scrollbar-track` | attachment list scrollbar track | `background-core-surface-subtle` | `#f1f1f1` | | `scrollbar-thumb` | attachment list scrollbar thumb | `border-core-default` | `#ddd` | +| `voice-color` | `VoiceWave` / `VoiceGlow` color when `colors` is unset | `accent-primary` | `#005fff` | +| `voice-backdrop` | what `VoiceWave` / `VoiceGlow` sit on (light or dark) | `background-core-elevation-0` | `#ffffff` | | `syntax-*` | code palette (see below) | none | built-in palette | The code palette tokens are `--str-chat__ai-syntax-text`, `-selection-bg`, `-comment`, `-punctuation`, `-number`, `-keyword`, `-tag`, `-string`, `-function`, `-cyan`, `-error` and `-whitespace`. @@ -298,6 +300,67 @@ function MessageContent({ content }) { } ``` +### `VoiceWave` and `VoiceGlow` + +Two animated visualizations of a voice agent, the web counterparts of `AIVoiceWave` and `AIVoiceGlow` in the Android and iOS AI components, with the same motion. `VoiceWave` is a single string carrying a wave packet that erupts with speech; `VoiceGlow` is a soft globe whose color masses orbit inside it and swell with speech. + +Both draw on a `` inside a `div` with `role="img"` and an accessible name such as "Voice agent: Speaking". They have no size of their own and draw in the middle of the box they are given (`VoiceWave` about 80% as wide as the shorter side, `VoiceGlow` about half), so size them with CSS. The defaults are `width: 100%; height: 8rem` for the wave and `12rem` square for the glow. + +#### Props + +| Name | Type | Required | Description | +| -------------- | ------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `state` | `'idle' \| 'connecting' \| 'listening' \| 'thinking' \| 'speaking'` | yes | What the agent is doing. The states differ only in motion, never in color, and a change eases from one motion into the next. | +| `colors` | `string[]` | no | One to five CSS colors, flowing through the shape. Past five are ignored. When omitted or empty, `--str-chat__ai-voice-color` is used, which follows the SDK accent color. | +| `expressivity` | `number` | no | From 0 to 2: how far and how fast it moves. Defaults to 1. | +| `audioLevel` | `() => number \| null \| undefined` | no | The level of whoever is talking, from 0 to 1, read once a frame, so a level changing many times a second re-renders nothing. Returning `null` (or omitting it) animates speech on its own. | +| `rendering` | `'auto' \| 'webgl' \| 'canvas'` | no | `VoiceGlow` only. `auto` (default) and `webgl` draw with a WebGL shader and fall back to a 2D canvas when WebGL is unavailable or its context is lost; `canvas` always uses the cheaper 2D drawing. | + +Any other `div` prop (`className`, `style`, event handlers, ...) is passed to the root. Pointer events are watched, never stopped: a tap inside the shape bounces it and a hovering mouse or pen ripples it, while your own `onClick` still fires. + +Behavior: + +- The animation runs only while the element is on screen and the tab is visible. +- With `prefers-reduced-motion: reduce`, the motion slows to a crawl and taps and hover are ignored. +- The palette and the light or dark look follow the theme (`--str-chat__ai-voice-color` and `--str-chat__ai-voice-backdrop`), re-read about once a second, so a theme switch needs no prop. +- Under SSR the markup is rendered and drawing starts after hydration. + +#### Example + +```tsx +import { useEffect, useState } from 'react'; +import { + useAudioLevel, + type VoiceAgentState, + VoiceGlow, + VoiceWave, +} from 'stream-chat-react/ai-components'; + +const VoiceAgent = ({ state }: { state: VoiceAgentState }) => { + const [mic, setMic] = useState(null); + const micLevel = useAudioLevel(mic); + + useEffect(() => { + let stream: MediaStream | undefined; + navigator.mediaDevices.getUserMedia({ audio: true }).then((granted) => { + stream = granted; + setMic(granted); + }); + return () => stream?.getTracks().forEach((track) => track.stop()); + }, []); + + return ( + <> + + + + ); +}; +``` + ### `StreamingMessage` The `StreamingMessage` is a component that displays text with a typewriter animation effect, similar to ChatGPT. It's ideal for streaming AI responses. It's a simplified wrapper with typewriter animation effect around the `AIMarkdown` component. @@ -507,6 +570,16 @@ function VoiceInput() { } ``` +### `useAudioLevel` + +Reads the live level of a `MediaStream`'s audio (a microphone from `getUserMedia({ audio: true })`, or a remote track) for the `audioLevel` prop of `VoiceWave` and `VoiceGlow`. + +```ts +const useAudioLevel: (stream: MediaStream | null | undefined) => () => number | null; +``` + +It returns a stable getter that measures the level when called, from 0 to 1, so nothing re-renders as the level changes. The getter returns `null` while there is no stream or no Web Audio support. The audio graph is torn down when the stream changes and on unmount; the stream's tracks are left running, since they belong to you. + ## Examples ### Basic Chat Interface diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx new file mode 100644 index 000000000..91186c1d2 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx @@ -0,0 +1,59 @@ +import { useCallback, useState } from 'react'; +import clsx from 'clsx'; + +import { createGlowCanvasRenderer, glowContains } from './renderers/glowCanvasRenderer'; +import { createGlowWebGLRenderer } from './renderers/glowWebGLRenderer'; +import type { VoiceGlowRendering, VoiceVisualizationProps } from './types'; +import { VoiceVisualizationView } from './VoiceVisualizationView'; + +export type VoiceGlowProps = VoiceVisualizationProps & { + /** How the glow is drawn. See `VoiceGlowRendering`. Defaults to `auto`. */ + rendering?: VoiceGlowRendering; +}; + +/** Whether `rendering` tries WebGL first; any WebGL failure still falls back to Canvas. */ +export const usesWebGL = (rendering: VoiceGlowRendering) => rendering !== 'canvas'; + +const createCanvasRenderer = (canvas: HTMLCanvasElement) => { + const ctx = canvas.getContext('2d'); + return ctx ? createGlowCanvasRenderer(ctx) : null; +}; + +/** + * A voice agent as a soft, airy globe: color masses orbit inside it and swell with speech, and + * the gaps between them let the background show through. Tapping the globe bounces it; drawn + * with WebGL, it also ripples from the touch and under a hovering mouse or pen. + * + * It has no size of its own and draws in the middle of the box it is given, about half as wide + * as its shorter side, so give it a square. + */ +export const VoiceGlow = ({ + className, + rendering = 'auto', + ...props +}: VoiceGlowProps) => { + // A canvas that has had a WebGL context can never give a 2D one, so falling back remounts it. + const [webGLFailed, setWebGLFailed] = useState(false); + const webGL = usesWebGL(rendering) && !webGLFailed; + + const createRenderer = useCallback( + (canvas: HTMLCanvasElement) => { + if (!webGL) return createCanvasRenderer(canvas); + const renderer = createGlowWebGLRenderer(canvas, () => setWebGLFailed(true)); + if (!renderer) setWebGLFailed(true); + return renderer; + }, + [webGL], + ); + + return ( + + ); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx new file mode 100644 index 000000000..2d1749946 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx @@ -0,0 +1,267 @@ +import { type PointerEvent, useEffect, useRef } from 'react'; +import clsx from 'clsx'; + +import { useTranslationContext } from '../../../context/TranslationContext'; +import { useReducedMotionPreference } from '../../../components/MessageList/hooks/useReducedMotionPreference'; +import { useStableCallback } from '../../../utils/useStableCallback'; +import { type VoiceDriverInput, VoiceVisualizationDriver } from './driver'; +import { audioLevelInput, expressivityInput } from './math'; +import { luminance, toHsvPalette } from './palette'; +import type { VoiceRenderer } from './renderers/types'; +import type { Hsv, VoiceAgentState, VoiceVisualizationProps } from './types'; + +/** Used when neither the `colors` prop nor the theme gives a readable color. */ +const FALLBACK_COLOR = '#005fff'; +/** How many frames pass between two reads of the theme, about once a second at 60 Hz. */ +const THEME_FRAMES = 60; +const MAX_PIXEL_RATIO = 2; + +type ViewProps = VoiceVisualizationProps & { + /** Whether a point, centred and normalized to the shorter side with y up, is on the shape. */ + contains: (x: number, y: number) => boolean; + /** Makes the renderer for a freshly mounted canvas, or null when it cannot draw there. */ + createRenderer: (canvas: HTMLCanvasElement) => VoiceRenderer | null; + /** Remounts the canvas and remakes the renderer when it changes. */ + rendererKey: string; +}; + +const useStateLabel = (state: VoiceAgentState) => { + const { t } = useTranslationContext(); + const labels: Record = { + connecting: t('Connecting'), + idle: t('Ready'), + listening: t('Listening'), + speaking: t('Speaking'), + thinking: t('Thinking'), + }; + return t('aria/Voice agent: {{ state }}', { state: labels[state] }); +}; + +/** + * What VoiceWave and VoiceGlow share: the motion advanced once a frame, the canvas kept at the + * element's size, the theme's colors, reduced motion, the pointer and the accessible name. + * + * The loop runs only while the element is on screen, reads `audioLevel` once a frame and + * redraws without re-rendering React. + */ +export const VoiceVisualizationView = ({ + audioLevel, + className, + colors, + contains, + createRenderer, + expressivity, + onPointerLeave, + onPointerMove, + onPointerUp, + rendererKey, + state, + ...restProps +}: ViewProps) => { + const label = useStateLabel(state); + const reduceMotion = useReducedMotionPreference(); + const rootRef = useRef(null); + const canvasRef = useRef(null); + const driverRef = useRef(null); + const audioLevelRef = useRef(audioLevel); + const themeRef = useRef<{ dark: number; palette: Hsv[] }>({ dark: 0, palette: [] }); + const paletteRef = useRef<{ key: string; palette: Hsv[] }>({ key: '', palette: [] }); + const propsRef = useRef({ colors, expressivity, reduceMotion, state }); + + /** + * The theme's accent color (the root's `color`) and whether it sits on a dark backdrop (the + * root's `text-decoration-color`, which carries the backdrop token and never paints). + */ + const readTheme = useStableCallback(() => { + const root = rootRef.current; + if (!root || typeof getComputedStyle !== 'function') return; + const style = getComputedStyle(root); + const backdrop = luminance(style.textDecorationColor || ''); + themeRef.current = { + dark: backdrop !== null && backdrop < 0.5 ? 1 : 0, + palette: toHsvPalette(style.color ? [style.color] : []), + }; + }); + + /** The driver's input from the latest props and the theme, keeping the audio level it has. */ + const applyInput = useStableCallback(() => { + // The first input settles the driver, so read the theme before it starts on a fallback. + if (!driverRef.current) readTheme(); + const { colors, expressivity, reduceMotion, state } = propsRef.current; + const key = (colors ?? []).join('|'); + if (paletteRef.current.key !== key) { + paletteRef.current = { key, palette: toHsvPalette(colors ?? []) }; + } + const own = paletteRef.current.palette; + const theme = themeRef.current.palette; + const input: VoiceDriverInput = { + audioLevel: driverRef.current?.input.audioLevel ?? null, + colors: own.length ? own : theme.length ? theme : toHsvPalette([FALLBACK_COLOR]), + expressivity: expressivityInput(expressivity), + reduceMotion, + state, + }; + if (driverRef.current) driverRef.current.input = input; + else + driverRef.current = new VoiceVisualizationDriver({ + ...input, + audioLevel: audioLevelInput(audioLevelRef.current?.()), + }); + }); + + useEffect(() => { + audioLevelRef.current = audioLevel; + propsRef.current = { colors, expressivity, reduceMotion, state }; + applyInput(); + }); + + const makeRenderer = useStableCallback(createRenderer); + + useEffect(() => { + const root = rootRef.current; + const canvas = canvasRef.current; + if (!root || !canvas || typeof requestAnimationFrame !== 'function') return; + + const renderer = makeRenderer(canvas); + if (!renderer) return; + + let pixelRatio = 1; + const resize = () => { + pixelRatio = Math.min(window.devicePixelRatio || 1, MAX_PIXEL_RATIO); + const width = Math.round(root.clientWidth * pixelRatio); + const height = Math.round(root.clientHeight * pixelRatio); + if (canvas.width !== width) canvas.width = width; + if (canvas.height !== height) canvas.height = height; + }; + resize(); + const resizeObserver = + typeof ResizeObserver === 'function' ? new ResizeObserver(resize) : null; + resizeObserver?.observe(root); + + let frameId = 0; + let last = 0; + let frames = 0; + let visible = true; + + const tick = (now: number) => { + frameId = 0; + const driver = driverRef.current; + if (!driver) return; + if (last !== 0) { + driver.input = { + ...driver.input, + audioLevel: audioLevelInput(audioLevelRef.current?.()), + }; + driver.advance((now - last) / 1000); + } + last = now; + if (++frames % THEME_FRAMES === 0) { + readTheme(); + applyInput(); + } + renderer.draw({ + dark: themeRef.current.dark, + dpr: pixelRatio, + driver, + height: canvas.height, + width: canvas.width, + }); + schedule(); + }; + + const schedule = () => { + if (visible && !frameId) frameId = requestAnimationFrame(tick); + }; + + const stop = () => { + if (frameId) cancelAnimationFrame(frameId); + frameId = 0; + // The first frame back shows where it was rather than leaping over the pause. + last = 0; + }; + + // Off screen, nothing is drawn; hidden tabs already pause requestAnimationFrame. + const intersectionObserver = + typeof IntersectionObserver === 'function' + ? new IntersectionObserver((entries) => { + visible = entries.some((entry) => entry.isIntersecting); + if (visible) schedule(); + else stop(); + }) + : null; + intersectionObserver?.observe(root); + + const colorScheme = + typeof window.matchMedia === 'function' + ? window.matchMedia('(prefers-color-scheme: dark)') + : null; + const onSchemeChange = () => { + readTheme(); + applyInput(); + }; + colorScheme?.addEventListener?.('change', onSchemeChange); + + schedule(); + + return () => { + stop(); + resizeObserver?.disconnect(); + intersectionObserver?.disconnect(); + colorScheme?.removeEventListener?.('change', onSchemeChange); + renderer.dispose?.(); + }; + }, [applyInput, makeRenderer, readTheme, rendererKey]); + + /** The pointer in the renderers' coordinates: centred, normalized to the shorter side, y up. */ + const toLocal = (event: PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + const side = Math.min(rect.width, rect.height) || 1; + return { + x: (event.clientX - rect.left - rect.width / 2) / side, + y: (rect.height / 2 - (event.clientY - rect.top)) / side, + }; + }; + + // Pointer events are only watched, never prevented or stopped, so a click handler on the + // element or a parent still gets them. + const handlePointerUp = (event: PointerEvent) => { + onPointerUp?.(event); + const { x, y } = toLocal(event); + if (contains(x, y)) driverRef.current?.tap(x, y); + }; + + const handlePointerMove = (event: PointerEvent) => { + onPointerMove?.(event); + // Only a hovering mouse or pen ripples it; a finger has no hover. + if (event.pointerType === 'touch') return; + const { x, y } = toLocal(event); + const hovering = event.buttons === 0 && contains(x, y); + driverRef.current?.hover(hovering ? x : null, hovering ? y : null); + }; + + const handlePointerLeave = (event: PointerEvent) => { + onPointerLeave?.(event); + driverRef.current?.hover(null, null); + }; + + return ( +
+
+ ); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceWave.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceWave.tsx new file mode 100644 index 000000000..e09c5d18b --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceWave.tsx @@ -0,0 +1,30 @@ +import clsx from 'clsx'; + +import { createWaveRenderer, waveContains } from './renderers/waveRenderer'; +import type { VoiceVisualizationProps } from './types'; +import { VoiceVisualizationView } from './VoiceVisualizationView'; + +const createRenderer = (canvas: HTMLCanvasElement) => { + const ctx = canvas.getContext('2d'); + return ctx ? createWaveRenderer(ctx) : null; +}; + +export type VoiceWaveProps = VoiceVisualizationProps; + +/** + * A voice agent as a single string, flat at its ends, carrying a wave packet that erupts with + * speech while its colors flow along it. Tapping the string ripples it from the touch, and so + * does a hovering mouse or pen. + * + * It has no size of its own and draws in the middle of the box it is given, about 80% as wide as + * its shorter side, so give it a square or a wide box. + */ +export const VoiceWave = ({ className, ...props }: VoiceWaveProps) => ( + +); diff --git a/src/plugins/AIComponents/VoiceVisualization/bounce.ts b/src/plugins/AIComponents/VoiceVisualization/bounce.ts new file mode 100644 index 000000000..a18656524 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/bounce.ts @@ -0,0 +1,56 @@ +const DURATION = 0.34; +const OFFSETS = [0, 0.3, 0.62, 1]; +const SCALES = [1, 0.93, 1.03, 1]; + +/** + * The bounce of a tap: a dip to 93% and an overshoot to 103% before settling, over 340 ms. + * + * These are authored keyframes rather than a spring: the ease-out curve times the whole run and + * the keyframes are linear between their stops. + */ +export class VoiceVisualizationBounce { + static readonly DURATION = DURATION; + + private age = Number.POSITIVE_INFINITY; + + start() { + this.age = 0; + } + + advance(seconds: number) { + this.age += seconds; + } + + get scale() { + if (this.age >= DURATION) return 1; + return scaleAt(easeOut(Math.max(this.age, 0) / DURATION)); + } +} + +export const scaleAt = (progress: number) => { + for (let i = 1; i < OFFSETS.length; i++) { + if (progress <= OFFSETS[i]) { + const fraction = (progress - OFFSETS[i - 1]) / (OFFSETS[i] - OFFSETS[i - 1]); + return SCALES[i - 1] + (SCALES[i] - SCALES[i - 1]) * fraction; + } + } + return 1; +}; + +/** CSS `ease-out`, `cubic-bezier(0, 0, 0.58, 1)`. */ +export const easeOut = (x: number) => { + const bx = 3 * 0.58; + const ax = 1 - bx; + const by = 3; + const ay = 1 - by; + const sampleX = (t: number) => (ax * t + bx) * t * t; + const sampleY = (t: number) => (ay * t + by) * t * t; + let low = 0; + let high = 1; + for (let i = 0; i < 24; i++) { + const middle = (low + high) / 2; + if (sampleX(middle) < x) low = middle; + else high = middle; + } + return sampleY((low + high) / 2); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/driver.ts b/src/plugins/AIComponents/VoiceVisualization/driver.ts new file mode 100644 index 000000000..d74a11fef --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/driver.ts @@ -0,0 +1,216 @@ +import { VoiceVisualizationBounce } from './bounce'; +import { MAXIMUM_COLORS } from './palette'; +import type { Hsv, VoiceAgentState } from './types'; + +export type VoiceDriverInput = { + /** One to five colors. */ + colors: Hsv[]; + /** From 0 to 2. */ + expressivity: number; + reduceMotion: boolean; + state: VoiceAgentState; + /** From 0 to 1, or null when no live audio drives the motion. */ + audioLevel?: number | null; +}; + +/** + * How a state moves. `load`, `flow` and `react` each select a motion pattern, one per state, so + * easing between states crossfades the patterns. + */ +type VoiceMotion = { + bright: number; + flow: number; + load: number; + react: number; + sat: number; + speed: number; +}; + +const motion = ({ + flow = 0, + load = 0, + react = 0, + speed, +}: Partial & { speed: number }): VoiceMotion => ({ + bright: 1, + flow, + load, + react, + sat: 1, + speed, +}); + +const MOTIONS: Record = { + connecting: motion({ load: 1, speed: 1 }), + idle: motion({ react: 0.12, speed: 0.4 }), + listening: motion({ react: 0.45, speed: 0.9 }), + speaking: motion({ react: 1, speed: 1.25 }), + thinking: motion({ flow: 1, speed: 1.25 }), +}; + +export const NO_TAP = 100; +const MAX_FRAME = 0.05; +const REDUCED = 0.07; +const DRIVERS = 0.08; +const GENTLE = 0.12; +const VOICE = 0.1; +const MIC = 0.35; + +/** Expressivity scales amplitude and speed, each with a floor so the motion breathes at 0. */ +export const scales = (expressivity: number) => ({ + amplitude: 0.15 + 0.85 * expressivity, + speed: 0.6 + 0.4 * expressivity, +}); + +/** + * The share of the way a value moves in `dt` seconds when it moves `perFrame` of the way each + * frame at 60 Hz. + */ +export const ease = (perFrame: number, dt: number) => 1 - Math.pow(1 - perFrame, dt * 60); + +const rem = (x: number, m: number) => ((x % m) + m) % m; + +/** Eases an HSV color, its hue taking the shorter way round the wheel. */ +export const easeHsv = (color: Hsv, target: Hsv, fraction: number): Hsv => { + const hueStep = (rem(target.hue - color.hue + 1.5, 1) - 0.5) * fraction; + return { + hue: rem(color.hue + hueStep + 1, 1), + saturation: color.saturation + (target.saturation - color.saturation) * fraction, + value: color.value + (target.value - color.value) * fraction, + }; +}; + +/** The five color slots the renderers read; past the palette they repeat its first color. */ +export const slots = (colors: Hsv[]): Hsv[] => { + const first = colors[0] ?? { hue: 0, saturation: 0, value: 0 }; + return Array.from({ length: MAXIMUM_COLORS }, (_, i) => colors[i] ?? first); +}; + +/** + * The motion of a voice visualization: where it is in time, how it moves, and its colors. + * + * Each value eases toward the state's motion by a fixed share per frame at 60 Hz, converted to + * time, so it moves the same at any refresh rate. The shader and the Canvas drawing read the + * same values, so the renderers move alike. + */ +export class VoiceVisualizationDriver { + speed: number; + bright: number; + sat: number; + load: number; + flow: number; + expressivity: number; + time = 0; + flowSpin = 0; + orbSpin = 0; + count: number; + tapX = 0; + tapY = 0; + tapAge = NO_TAP; + hoverX = 0; + hoverY = 0; + hoverAmount = 0; + mic = 0; + voice = 0; + + private baseReact: number; + private colors: Hsv[]; + private isHovering = false; + private bounce = new VoiceVisualizationBounce(); + private currentInput: VoiceDriverInput; + + // Starts settled on the input, so the first frame shows it rather than easing into it. + constructor(input: VoiceDriverInput) { + this.currentInput = input; + const m = MOTIONS[input.state]; + const s = scales(input.expressivity); + this.speed = m.speed * s.speed; + this.bright = m.bright; + this.sat = m.sat; + this.load = m.load * s.amplitude; + this.flow = m.flow * s.amplitude; + this.baseReact = m.react * s.amplitude; + this.expressivity = input.expressivity; + this.colors = slots(input.colors); + this.count = input.colors.length; + } + + get input() { + return this.currentInput; + } + + set input(value: VoiceDriverInput) { + this.currentInput = value; + if (value.reduceMotion) this.isHovering = false; + } + + /** Live audio lifts the motion to its level, whatever the state. */ + get react() { + return ( + this.baseReact + (Math.max(this.baseReact, this.mic) - this.baseReact) * this.voice + ); + } + + /** The scale of a tap's bounce. */ + get scale() { + return this.bounce.scale; + } + + /** Color slot `index`, 0 to 4; past the palette the slots repeat its first color. */ + color(index: number) { + return this.colors[index]; + } + + tap(x: number, y: number) { + if (this.currentInput.reduceMotion) return; + this.tapX = x; + this.tapY = y; + this.tapAge = 0; + this.bounce.start(); + } + + /** Where a pointer hovers, or null once it has left. */ + hover(x: number | null, y: number | null) { + if (x === null || y === null || this.currentInput.reduceMotion) { + this.isHovering = false; + return; + } + this.hoverX = x; + this.hoverY = y; + this.isHovering = true; + } + + advance(seconds: number) { + // Clamped so a frame after a stall cannot leap the motion. + const dt = Math.min(Math.max(seconds, 0), MAX_FRAME); + const input = this.currentInput; + const m = MOTIONS[input.state]; + const s = scales(input.expressivity); + const drivers = ease(DRIVERS, dt); + this.speed += (m.speed * s.speed - this.speed) * drivers; + this.bright += (m.bright - this.bright) * drivers; + this.sat += (m.sat - this.sat) * drivers; + this.load += (m.load * s.amplitude - this.load) * drivers; + this.flow += (m.flow * s.amplitude - this.flow) * drivers; + this.baseReact += (m.react * s.amplitude - this.baseReact) * drivers; + this.expressivity += (input.expressivity - this.expressivity) * drivers; + // Integrated rather than computed as time times a weight, so a change of state eases the + // turns instead of jumping their angles. + const step = dt * (input.reduceMotion ? REDUCED : 1) * this.speed; + this.time += step; + this.flowSpin += step * this.flow; + this.orbSpin += step * (1 + 0.5 * this.flow + 0.1 * this.baseReact); + this.tapAge += dt; + this.bounce.advance(dt); + const gentle = ease(GENTLE, dt); + this.hoverAmount += ((this.isHovering ? 1 : 0) - this.hoverAmount) * gentle; + const level = input.audioLevel ?? null; + this.voice += ((level === null ? 0 : 1) - this.voice) * ease(VOICE, dt); + this.mic += ((level ?? 0) - this.mic) * ease(MIC, dt); + const targets = slots(input.colors); + for (let i = 0; i < this.colors.length; i++) { + this.colors[i] = easeHsv(this.colors[i], targets[i], gentle); + } + this.count += (input.colors.length - this.count) * gentle; + } +} diff --git a/src/plugins/AIComponents/VoiceVisualization/index.ts b/src/plugins/AIComponents/VoiceVisualization/index.ts new file mode 100644 index 000000000..2bca4b689 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/index.ts @@ -0,0 +1,8 @@ +export { VoiceGlow, type VoiceGlowProps } from './VoiceGlow'; +export { VoiceWave, type VoiceWaveProps } from './VoiceWave'; +export { useAudioLevel } from './useAudioLevel'; +export type { + VoiceAgentState, + VoiceGlowRendering, + VoiceVisualizationProps, +} from './types'; diff --git a/src/plugins/AIComponents/VoiceVisualization/math.ts b/src/plugins/AIComponents/VoiceVisualization/math.ts new file mode 100644 index 000000000..e49d7e54b --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/math.ts @@ -0,0 +1,68 @@ +import type { Hsv } from './types'; + +/** The shader math the Canvas renderers run on the CPU, once per frame. */ + +export const mix = (a: number, b: number, t: number) => a + (b - a) * t; + +export const fract = (x: number) => x - Math.floor(x); + +export const clamp = (x: number, low: number, high: number) => + Math.min(Math.max(x, low), high); + +/** GLSL's smoothstep, evaluated the same with its edges either way round. */ +export const smoothstep = (edge0: number, edge1: number, x: number) => { + const t = clamp((x - edge0) / (edge1 - edge0), 0, 1); + return t * t * (3 - 2 * t); +}; + +/** HSV, hue a fraction of a turn, into `out` as RGB. */ +export const hsvToRgb = ( + hue: number, + saturation: number, + value: number, + out: number[], +) => { + for (let i = 0; i < 3; i++) { + const k = i === 0 ? 1 : i === 1 ? 2 / 3 : 1 / 3; + const p = Math.abs(fract(hue + k) * 6 - 3); + out[i] = value * mix(1, clamp(p - 1, 0, 1), saturation); + } +}; + +/** A palette color as is; dark mode lifts saturation a touch so it stays vivid on black. */ +export const vivid = (color: Hsv, dark: number, out: number[]) => + hsvToRgb(color.hue, clamp(color.saturation + dark * 0.06, 0, 1), color.value, out); + +/** Pushes `c` away from its luminance by `amount`; below 1 desaturates. */ +export const saturate = (c: number[], amount: number) => { + const l = 0.299 * c[0] + 0.587 * c[1] + 0.114 * c[2]; + for (let i = 0; i < 3; i++) c[i] = clamp(mix(l, c[i], amount), 0, 1); +}; + +const gradient = (i: number, seed: number) => + fract(Math.sin(i * 127.1 + seed * 311.7) * 43758.547) * 2 - 1; + +/** Smooth 1-D gradient noise from -1 to 1: the slow wobble the CPU needs, where the shader uses simplex. */ +export const noise = (x: number, seed: number) => { + const i = Math.floor(x); + const f = x - i; + const u = f * f * (3 - 2 * f); + return mix(gradient(i, seed) * f, gradient(i + 1, seed) * (f - 1), u) * 2; +}; + +/** How much the agent is talking, 0 to 1: a slow phrase gate times a faster syllable wobble. */ +export const speechEnv = (t: number) => { + const gate = smoothstep(-0.25, 0.35, noise(t * 0.6, 21)); + const syllable = 0.55 + 0.45 * noise(t * 3, 5); + return clamp(gate * syllable, 0, 1); +}; + +/** `expressivity` from 0 to 2; one that is not a number gets the default, 1. */ +export const expressivityInput = (expressivity: number | undefined) => + typeof expressivity === 'number' && Number.isFinite(expressivity) + ? clamp(expressivity, 0, 2) + : 1; + +/** `level` from 0 to 1; one that is not a number counts as no live audio, so it can't stall the motion. */ +export const audioLevelInput = (level: number | null | undefined) => + typeof level === 'number' && Number.isFinite(level) ? clamp(level, 0, 1) : null; diff --git a/src/plugins/AIComponents/VoiceVisualization/palette.ts b/src/plugins/AIComponents/VoiceVisualization/palette.ts new file mode 100644 index 000000000..3ec6e141d --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/palette.ts @@ -0,0 +1,91 @@ +import type { Hsv } from './types'; + +export const MAXIMUM_COLORS = 5; + +type Rgb = [number, number, number]; + +let normalizer: CanvasRenderingContext2D | null | undefined; + +/** + * The browser's own reading of a CSS color, as `#rrggbb` or `rgba(...)`, so any syntax it + * understands (names, `hsl()`, `color-mix()`, ...) reaches the parser below in one of two forms. + */ +const normalize = (color: string) => { + if (normalizer === undefined) { + normalizer = + typeof document === 'undefined' + ? null + : document.createElement('canvas').getContext('2d'); + } + if (!normalizer) return color; + // An invalid color leaves the previous value in place, so reset first and compare. + normalizer.fillStyle = '#000000'; + normalizer.fillStyle = color; + return String(normalizer.fillStyle); +}; + +const channel = (value: string) => + value.trim().endsWith('%') ? parseFloat(value) / 100 : parseFloat(value) / 255; + +/** Reads `#rgb`, `#rrggbb` (with optional alpha) and `rgb()`/`rgba()` into 0-1 channels. */ +export const parseColor = (color: string): Rgb | null => { + const value = normalize(color.trim()).trim().toLowerCase(); + const hex = /^#([0-9a-f]{3,8})$/.exec(value)?.[1]; + if (hex) { + const full = + hex.length <= 4 + ? hex + .slice(0, 3) + .split('') + .map((c) => c + c) + .join('') + : hex.slice(0, 6); + if (full.length !== 6) return null; + return [0, 2, 4].map((i) => parseInt(full.slice(i, i + 2), 16) / 255) as Rgb; + } + const rgb = /^rgba?\(([^)]+)\)$/.exec(value)?.[1]; + if (rgb) { + const parts = rgb.split(/[\s,/]+/).filter(Boolean); + if (parts.length < 3) return null; + const channels = parts.slice(0, 3).map(channel); + if (channels.some((c) => !Number.isFinite(c))) return null; + return channels.map((c) => Math.min(Math.max(c, 0), 1)) as Rgb; + } + return null; +}; + +/** RGB channels from 0 to 1 as HSV, keeping full saturation and value so muted colors render true. */ +export const rgbToHsv = ([r, g, b]: Rgb): Hsv => { + const high = Math.max(r, g, b); + const low = Math.min(r, g, b); + const delta = high - low; + let hue = 0; + if (delta !== 0) { + if (high === r) hue = ((((g - b) / delta) % 6) + 6) % 6; + else if (high === g) hue = (b - r) / delta + 2; + else hue = (r - g) / delta + 4; + } + return { + hue: (((hue / 6) % 1) + 1) % 1, + saturation: high === 0 ? 0 : Math.min(Math.max(delta / high, 0), 1), + value: Math.min(Math.max(high, 0), 1), + }; +}; + +/** CSS colors as the HSV the renderers read, at most five; colors that cannot be read are skipped. */ +export const toHsvPalette = (colors: readonly string[]): Hsv[] => + colors + .map(parseColor) + .filter((rgb): rgb is Rgb => rgb !== null) + .slice(0, MAXIMUM_COLORS) + .map(rgbToHsv); + +/** Relative luminance of a CSS color, or null when it cannot be read. */ +export const luminance = (color: string) => { + const rgb = parseColor(color); + if (!rgb) return null; + const [r, g, b] = rgb.map((c) => + c <= 0.03928 ? c / 12.92 : Math.pow((c + 0.055) / 1.055, 2.4), + ); + return 0.2126 * r + 0.7152 * g + 0.0722 * b; +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts b/src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts new file mode 100644 index 000000000..d4f95bd30 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts @@ -0,0 +1,214 @@ +import type { VoiceVisualizationDriver } from '../driver'; +import { clamp, mix, noise, saturate, smoothstep, speechEnv, vivid } from '../math'; +import { rgba, type VoiceFrame, type VoiceRenderer } from './types'; + +const REACH = [1.15, 1.3, 1.2, 1.25]; + +/** How far from the centre a tap still lands on the globe, as a share of the shorter side. */ +const GLOBE_REACH = 0.32; + +/** Whether a point, centred and normalized to the shorter side with y up, lands on the globe. */ +export const glowContains = (x: number, y: number) => Math.hypot(x, y) < GLOBE_REACH; + +/** + * The glow drawn with a handful of gradients: color masses orbiting inside a soft globe, a + * highlight, a rim and a halo. Cheaper than the shader and drawn wherever a 2D canvas is; it + * moves the same but has none of the shader's noise-warped marbling or ripples. + */ +export const createGlowCanvasRenderer = ( + ctx: CanvasRenderingContext2D, +): VoiceRenderer => { + const base = [0, 0, 0]; + const k0 = [0, 0, 0]; + const colB = [0, 0, 0]; + const colC = [0, 0, 0]; + const colD = [0, 0, 0]; + const colE = [0, 0, 0]; + const halo = [0, 0, 0]; + const rim = [0, 0, 0]; + const tmp = [0, 0, 0]; + const tmp2 = [0, 0, 0]; + + /** Slot `index` fading in over a lighter or darker first color as the palette grows. */ + const mixSlot = ( + d: VoiceVisualizationDriver, + index: number, + dark: number, + lighter: boolean, + into: number[], + ) => { + for (let i = 0; i < 3; i++) tmp2[i] = clamp(tmp[i] * (lighter ? 1.18 : 0.85), 0, 1); + saturate(tmp2, 1.3); + vivid(d.color(index), dark, into); + saturate(into, 1.7); + const present = clamp(d.count - index, 0, 1); + for (let i = 0; i < 3; i++) into[i] = mix(tmp2[i], into[i], present); + }; + + const colors = (d: VoiceVisualizationDriver, dark: number) => { + vivid(d.color(0), dark, tmp); + for (let i = 0; i < 3; i++) k0[i] = tmp[i]; + saturate(k0, 1.7); + for (let i = 0; i < 3; i++) base[i] = mix(tmp[i], 1, 0.06); + saturate(base, 1.3); + for (let i = 0; i < 3; i++) + halo[i] = mix(mix(tmp[i], 1, 0.75), mix(tmp[i], 1, 0.32), dark); + for (let i = 0; i < 3; i++) rim[i] = mix(1, k0[i], 0.6); + mixSlot(d, 1, dark, true, colB); + mixSlot(d, 2, dark, false, colC); + vivid(d.color(3), dark, colD); + saturate(colD, 1.7); + vivid(d.color(4), dark, colE); + saturate(colE, 1.7); + }; + + const circle = ( + x: number, + y: number, + radius: number, + fill: CanvasGradient | string, + ) => { + ctx.beginPath(); + ctx.arc(x, y, Math.max(radius, 0), 0, Math.PI * 2); + ctx.fillStyle = fill; + ctx.fill(); + }; + + const radial = ( + x: number, + y: number, + radius: number, + stops: Array<[number, string]>, + ) => { + const gradient = ctx.createRadialGradient(x, y, 0, x, y, Math.max(radius, 1e-3)); + for (const [offset, color] of stops) + gradient.addColorStop(clamp(offset, 0, 1), color); + return gradient; + }; + + const withScale = (scale: number, cx: number, cy: number, draw: () => void) => { + ctx.save(); + ctx.translate(cx, cy); + ctx.scale(scale, scale); + ctx.translate(-cx, -cy); + draw(); + ctx.restore(); + }; + + const draw = ({ dark, driver: d, height, width }: VoiceFrame) => { + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.globalCompositeOperation = 'source-over'; + ctx.clearRect(0, 0, width, height); + const side = Math.min(width, height); + if (side <= 0) return; + const cx = width / 2; + const cy = height / 2; + const t = d.time; + const react = d.react; + const speak = smoothstep(0.6, 1, react); + const wobble = 0.55 + 0.25 * Math.sin(t * 1.6); + const speech = mix(react * mix(wobble, speechEnv(t), speak), d.mic, d.voice); + const speechSmooth = mix(speech, react * wobble, 0.5 * speak); + const energy = + 0.12 + 0.3 * react + 0.18 * d.flow + 0.28 * d.load + 0.06 * speechSmooth; + const r = + 0.24 * + side * + (1 + + speechSmooth * 0.2 + + d.load * 0.05 * Math.sin(t * 2) + + react * 0.04 * Math.sin(t * 1.6)); + const gain = d.bright * mix(1, 0.45 + 0.35 * Math.sin(t * 2), d.load); + colors(d, dark); + + const blob = (i: number, color: number[], alpha: number) => { + if (alpha <= 0.002) return; + const angleByIndex = [ + d.orbSpin * 0.42 + 2 * noise(t * 0.12, 0), + -d.orbSpin * 0.34 + 2 * noise(t * 0.1, 5) + 2, + d.orbSpin * 0.3 + 2 * noise(t * 0.11, 9) + 1, + -d.orbSpin * 0.26 + 2 * noise(t * 0.09, 14) + 3.5, + ]; + const angle = angleByIndex[i] + d.flowSpin * 0.3; + const distance = 0.4 - 0.02 * i; + // A slow breath stands in for the shader's per-pixel warp. + const blobRadius = REACH[i] * r * (1 + 0.12 * energy * noise(t * 0.5 + i, 33)); + const x = cx + distance * r * Math.cos(angle); + const y = cy - distance * r * Math.sin(angle); + circle( + x, + y, + blobRadius, + radial(x, y, blobRadius, [ + [0, rgba(color, alpha)], + [0.09, rgba(color, alpha)], + [0.55, rgba(color, alpha * 0.5)], + [1, rgba(color, 0)], + ]), + ); + }; + + withScale(d.scale, cx, cy, () => { + circle(cx, cy, r * 1.06, rgba(base, 0.15 * gain)); + const blobAlpha = 0.6 * (0.75 + 0.25 * energy) * gain; + blob(1, colB, blobAlpha * 0.8); + blob(0, k0, blobAlpha); + blob(0, colC, blobAlpha * 0.5); + blob(2, colD, blobAlpha * clamp(d.count - 3, 0, 1)); + blob(3, colE, blobAlpha * clamp(d.count - 4, 0, 1)); + // The highlight and the rim. + circle( + cx, + cy, + r * 1.06, + radial(cx - 0.35 * r, cy - 0.45 * r, r, [ + [0, `rgba(255, 255, 255, ${clamp(0.16 * gain, 0, 1)})`], + [1, 'rgba(255, 255, 255, 0)'], + ]), + ); + circle( + cx, + cy, + r * 1.06, + radial(cx, cy, r * 1.06, [ + [0.7, rgba(rim, 0)], + [0.95, rgba(rim, 0.1 * gain)], + [1, rgba(rim, 0)], + ]), + ); + }); + + // The soft edge, outside the bounce's scale so it covers the whole area at any scale and + // whatever spilled past the globe goes too. The canvas is the glow's own layer, so this masks + // only the glow, never what is behind it. + ctx.globalCompositeOperation = 'destination-in'; + ctx.fillStyle = radial(cx, cy, r * 1.06 * d.scale, [ + [0, 'rgba(0, 0, 0, 1)'], + [0.62 / 1.06, 'rgba(0, 0, 0, 1)'], + [1, 'rgba(0, 0, 0, 0)'], + ]); + ctx.fillRect(0, 0, width, height); + + // The halo, behind the globe. + ctx.globalCompositeOperation = 'destination-over'; + withScale(d.scale, cx, cy, () => { + const alpha = (mix(0.05, 0.1, dark) + 0.05 * react) * gain; + const outer = r * 0.8 + 0.25 * side; + const inner = (r * 0.8) / outer; + circle( + cx, + cy, + outer, + radial(cx, cy, outer, [ + [0, rgba(halo, alpha)], + [inner, rgba(halo, alpha)], + [inner + (0.09 * side) / outer, rgba(halo, alpha * Math.exp(-1))], + [1, rgba(halo, 0)], + ]), + ); + }); + ctx.globalCompositeOperation = 'source-over'; + }; + + return { draw }; +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/renderers/glowWebGLRenderer.ts b/src/plugins/AIComponents/VoiceVisualization/renderers/glowWebGLRenderer.ts new file mode 100644 index 000000000..b2946ab3c --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/renderers/glowWebGLRenderer.ts @@ -0,0 +1,365 @@ +import type { VoiceFrame, VoiceRenderer } from './types'; + +const VERTEX = ` +attribute vec2 position; +void main() { + gl_Position = vec4(position, 0.0, 1.0); +} +`; + +/** + * The glow's fragment shader, GLSL ES 1.00. A line-for-line port of the AGSL glow in + * stream-chat-android (itself ported from the iOS Metal glow), so the platforms look alike. + * Pixels past the halo return at once, so only the glow itself costs. + */ +const FRAGMENT = ` +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +#else +precision mediump float; +#endif + +uniform vec2 resolution; +uniform vec4 col0; +uniform vec4 col1; +uniform vec4 col2; +uniform vec4 col3; +uniform vec4 col4; +uniform float time; +uniform float count; +uniform float bright; +uniform float sat; +uniform float load; +uniform float flow; +uniform float flowSpin; +uniform float orbSpin; +uniform float react; +uniform float dark; +uniform float mic; +uniform float voice; +uniform float scale; +uniform vec2 tap; +uniform float tapTime; +uniform vec2 hover; +uniform float hoverAmt; + +float sq(float x) { return x * x; } +float ss(float e0, float e1, float x) { + float t = clamp((x - e0) / (e1 - e0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} +vec3 hsv2rgb(vec3 c) { + vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0); + vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www); + return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y); +} +vec3 desat(vec3 c, float s) { + float l = dot(c, vec3(0.299, 0.587, 0.114)); + return mix(vec3(l), c, s); +} +vec3 saturate3(vec3 c, float s) { + float l = dot(c, vec3(0.299, 0.587, 0.114)); + return clamp(mix(vec3(l), c, s), 0.0, 1.0); +} +vec2 rot2(vec2 p, float a) { + float s = sin(a); float c = cos(a); + return vec2(c * p.x + s * p.y, -s * p.x + c * p.y); +} +vec3 vivid(vec4 hsv) { + return hsv2rgb(vec3(hsv.x, clamp(hsv.y + dark * 0.06, 0.0, 1.0), hsv.z)); +} +// Waves spreading out from the last tap, fading within a second and a half. +float tapRipple(vec2 p) { + float age = tapTime; + if (age > 1.5) return 0.0; + float d = distance(p, tap); + float front = age * 0.75; + float env = exp(-sq((d - front) / 0.14)); + float decay = exp(-age * 2.6); + return sin(d * 34.0 - age * 24.0) * env * decay; +} +// A gentle ripple following a hovering pointer. +float hoverRipple(vec2 p) { + if (hoverAmt <= 0.001) return 0.0; + float d = distance(p, hover); + float env = exp(-sq(d / 0.22)); + return sin(d * 30.0 - time * 7.0) * env * hoverAmt * 0.6; +} +vec4 permute(vec4 x) { return mod(((x * 34.0) + 1.0) * x, 289.0); } +vec4 taylorInvSqrt(vec4 r) { return 1.79284291400159 - 0.85373472095314 * r; } +float snoise(vec3 v) { + const vec2 C = vec2(1.0 / 6.0, 1.0 / 3.0); + const vec4 D = vec4(0.0, 0.5, 1.0, 2.0); + vec3 i = floor(v + dot(v, C.yyy)); + vec3 x0 = v - i + dot(i, C.xxx); + vec3 g = step(x0.yzx, x0.xyz); + vec3 l = 1.0 - g; + vec3 i1 = min(g.xyz, l.zxy); + vec3 i2 = max(g.xyz, l.zxy); + vec3 x1 = x0 - i1 + 1.0 * C.xxx; + vec3 x2 = x0 - i2 + 2.0 * C.xxx; + vec3 x3 = x0 - 1.0 + 3.0 * C.xxx; + i = mod(i, 289.0); + vec4 p = permute(permute(permute( + i.z + vec4(0.0, i1.z, i2.z, 1.0)) + + i.y + vec4(0.0, i1.y, i2.y, 1.0)) + + i.x + vec4(0.0, i1.x, i2.x, 1.0)); + float n_ = 1.0 / 7.0; + vec3 ns = n_ * D.wyz - D.xzx; + vec4 j = p - 49.0 * floor(p * ns.z * ns.z); + vec4 x_ = floor(j * ns.z); + vec4 y_ = floor(j - 7.0 * x_); + vec4 x = x_ * ns.x + ns.yyyy; + vec4 y = y_ * ns.x + ns.yyyy; + vec4 h = 1.0 - abs(x) - abs(y); + vec4 b0 = vec4(x.xy, y.xy); + vec4 b1 = vec4(x.zw, y.zw); + vec4 s0 = floor(b0) * 2.0 + 1.0; + vec4 s1 = floor(b1) * 2.0 + 1.0; + vec4 sh = -step(h, vec4(0.0)); + vec4 a0 = b0.xzyw + s0.xzyw * sh.xxyy; + vec4 a1 = b1.xzyw + s1.xzyw * sh.zzww; + vec3 p0 = vec3(a0.xy, h.x); + vec3 p1 = vec3(a0.zw, h.y); + vec3 p2 = vec3(a1.xy, h.z); + vec3 p3 = vec3(a1.zw, h.w); + vec4 norm = taylorInvSqrt(vec4(dot(p0, p0), dot(p1, p1), dot(p2, p2), dot(p3, p3))); + p0 *= norm.x; p1 *= norm.y; p2 *= norm.z; p3 *= norm.w; + vec4 m = max(0.6 - vec4(dot(x0, x0), dot(x1, x1), dot(x2, x2), dot(x3, x3)), 0.0); + m = m * m; + return 42.0 * dot(m * m, vec4(dot(p0, x0), dot(p1, x1), dot(p2, x2), dot(p3, x3))); +} +float speechEnv(float t) { + float gate = ss(-0.25, 0.35, snoise(vec3(t * 0.6, 21.0, 0.0))); + float syl = 0.55 + 0.45 * snoise(vec3(t * 3.0, 5.0, 0.0)); + return clamp(gate * syl, 0.0, 1.0); +} + +void main() { + // Centred, normalized to the shorter side, y up: gl_FragCoord already has y up. + float side = min(resolution.x, resolution.y); + vec2 q = (gl_FragCoord.xy - 0.5 * resolution) / side / scale; + float t = time; + float r = length(q); + + float speak = ss(0.6, 1.0, react); + float speech = react * mix(0.55 + 0.25 * sin(t * 1.6), speechEnv(t), speak); + speech = mix(speech, mic, voice); + float speechSmooth = mix(speech, react * (0.55 + 0.25 * sin(t * 1.6)), 0.5 * speak); + float energy = 0.12 + 0.30 * react + 0.18 * flow + 0.28 * load + 0.06 * speechSmooth; + float R = 0.24 * (1.0 + speechSmooth * 0.20 + load * 0.05 * sin(t * 2.0) + react * 0.04 * sin(t * 1.6)); + float p = r / R; + if (p > 2.2) { + gl_FragColor = vec4(0.0); + return; + } + + float core = exp(-2.0 * p * p); + float body = ss(1.06, 0.62, p); + + vec2 P = q / R; + float pl = length(P); + float nt = t * 0.15 + flowSpin * 0.05; + float nAmp = 0.20 + 0.28 * energy; + vec2 sp = P + nAmp * vec2(snoise(vec3(P * 0.7, nt)), snoise(vec3(P * 0.7 + 4.7, nt))); + float ang = atan(P.y, P.x); + float wave = (0.018 + 0.025 * energy) * sin(pl * 5.0 - t * 2.0) * ss(0.0, 0.4, pl); + sp += wave * vec2(-sin(ang), cos(ang)); + sp = rot2(sp, flowSpin * 0.30); + vec2 gtoP = q - (hoverAmt > 0.001 ? hover : tap); + sp += (gtoP / (length(gtoP) + 1e-4)) * (tapRipple(q) + hoverRipple(q)) * 0.5; + float bt = t; + float a0 = orbSpin * 0.42 + 2.0 * snoise(vec3(bt * 0.12, 0.0, 0.0)); + float a1 = -orbSpin * 0.34 + 2.0 * snoise(vec3(bt * 0.10, 5.0, 0.0)) + 2.0; + float a2 = orbSpin * 0.30 + 2.0 * snoise(vec3(bt * 0.11, 9.0, 0.0)) + 1.0; + float a3 = -orbSpin * 0.26 + 2.0 * snoise(vec3(bt * 0.09, 14.0, 0.0)) + 3.5; + vec2 c0 = 0.40 * vec2(cos(a0), sin(a0)); + vec2 c1 = 0.38 * vec2(cos(a1), sin(a1)); + vec2 c2c = 0.36 * vec2(cos(a2), sin(a2)); + vec2 c3c = 0.34 * vec2(cos(a3), sin(a3)); + float b0 = ss(1.15, 0.1, length(sp - c0)); + float b1 = ss(1.30, 0.1, length(sp - c1)); + float b2 = ss(1.20, 0.1, length(sp - c2c)); + float b3 = ss(1.25, 0.1, length(sp - c3c)); + + vec3 k0 = saturate3(vivid(col0), 1.7); + vec3 kL = saturate3(clamp(vivid(col0) * 1.18, 0.0, 1.0), 1.3); + vec3 kD = saturate3(vivid(col0) * 0.85, 1.3); + vec3 colB = mix(kL, saturate3(vivid(col1), 1.7), clamp(count - 1.0, 0.0, 1.0)); + vec3 colC = mix(kD, saturate3(vivid(col2), 1.7), clamp(count - 2.0, 0.0, 1.0)); + vec3 colD = saturate3(vivid(col3), 1.7); + vec3 colE = saturate3(vivid(col4), 1.7); + vec3 base = saturate3(mix(vivid(col0), vec3(1.0), 0.06), 1.3); + vec3 col = base; + col = mix(col, k0, b0); + col = mix(col, colB, b1); + col = mix(col, colC, b0 * 0.5); + col = mix(col, colD, b2 * clamp(count - 3.0, 0.0, 1.0)); + col = mix(col, colE, b3 * clamp(count - 4.0, 0.0, 1.0)); + col = saturate3(col, 1.15); + col = clamp(col * 1.12, 0.0, 1.0); + col = desat(col, sat); + + float z = sqrt(max(1.0 - p * p, 0.0)); + vec3 nrm = normalize(vec3(q / R, z + 1e-4)); + vec3 Ld = normalize(vec3(-0.35, 0.45, 0.85)); + float diff = clamp(dot(nrm, Ld), 0.0, 1.0); + col *= 0.80 + 0.20 * diff; + float fres = pow(1.0 - clamp(nrm.z, 0.0, 1.0), 2.5); + col += fres * 0.10 * mix(vec3(1.0), k0, 0.6); + + vec3 haloCol = mix(mix(vivid(col0), vec3(1.0), 0.75), mix(vivid(col0), vec3(1.0), 0.32), dark); + float halo = exp(-sq(max(r - R * 0.8, 0.0) / 0.09)); + float haloA = halo * (mix(0.05, 0.10, dark) + 0.05 * react); + float waves = clamp(b0 + 0.8 * b1, 0.0, 1.0); + float bodyA = clamp(body * (0.15 + 0.60 * waves + 0.06 * core) * (0.92 + 0.08 * fres), 0.0, 1.0); + float fade = mix(1.0, 0.45 + 0.35 * sin(t * 2.0), load); + vec3 pm = col * bodyA + haloCol * haloA * (1.0 - bodyA); + float al = bodyA + haloA * (1.0 - bodyA); + float gain = bright * fade; + // Premultiplied, as the context is. + gl_FragColor = vec4(pm * gain, al * gain); +} +`; + +const UNIFORMS = [ + 'resolution', + 'col0', + 'col1', + 'col2', + 'col3', + 'col4', + 'time', + 'count', + 'bright', + 'sat', + 'load', + 'flow', + 'flowSpin', + 'orbSpin', + 'react', + 'dark', + 'mic', + 'voice', + 'scale', + 'tap', + 'tapTime', + 'hover', + 'hoverAmt', +] as const; + +type Uniform = (typeof UNIFORMS)[number]; + +const compile = (gl: WebGLRenderingContext, type: number, source: string) => { + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + gl.deleteShader(shader); + return null; + } + return shader; +}; + +/** + * The glow drawn by a WebGL shader, every pixel computed on the GPU: color masses warped by + * noise, concentric waves, ripples from a tap or a hovering pointer, light and a halo. It reads + * the same motion as the Canvas glow. + * + * Returns null when WebGL is unavailable or the shader does not compile, so the caller can fall + * back to Canvas. `onLost` is called if the browser later drops the context. + */ +export const createGlowWebGLRenderer = ( + canvas: HTMLCanvasElement, + onLost: () => void, +): VoiceRenderer | null => { + let gl: WebGLRenderingContext | null = null; + try { + gl = canvas.getContext('webgl', { + alpha: true, + antialias: false, + depth: false, + premultipliedAlpha: true, + preserveDrawingBuffer: false, + stencil: false, + }); + } catch { + gl = null; + } + if (!gl) return null; + + const vertex = compile(gl, gl.VERTEX_SHADER, VERTEX); + const fragment = compile(gl, gl.FRAGMENT_SHADER, FRAGMENT); + const program = gl.createProgram(); + if (!vertex || !fragment || !program) return null; + gl.attachShader(program, vertex); + gl.attachShader(program, fragment); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) return null; + gl.useProgram(program); + + // One triangle covering the whole viewport. + const buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, 3, -1, -1, 3]), + gl.STATIC_DRAW, + ); + const position = gl.getAttribLocation(program, 'position'); + gl.enableVertexAttribArray(position); + gl.vertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0); + + const locations = {} as Record; + for (const name of UNIFORMS) locations[name] = gl.getUniformLocation(program, name); + + let lost = false; + const handleLost = (event: Event) => { + event.preventDefault(); + lost = true; + onLost(); + }; + canvas.addEventListener('webglcontextlost', handleLost); + + const draw = ({ dark, driver: d, height, width }: VoiceFrame) => { + if (lost || !gl) return; + gl.viewport(0, 0, width, height); + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.uniform2f(locations.resolution, width, height); + for (let i = 0; i < 5; i++) { + const c = d.color(i); + gl.uniform4f(locations[`col${i}` as Uniform], c.hue, c.saturation, c.value, 0); + } + gl.uniform1f(locations.time, d.time); + gl.uniform1f(locations.count, d.count); + gl.uniform1f(locations.bright, d.bright); + gl.uniform1f(locations.sat, d.sat); + gl.uniform1f(locations.load, d.load); + gl.uniform1f(locations.flow, d.flow); + gl.uniform1f(locations.flowSpin, d.flowSpin); + gl.uniform1f(locations.orbSpin, d.orbSpin); + gl.uniform1f(locations.react, d.react); + gl.uniform1f(locations.dark, dark); + gl.uniform1f(locations.mic, d.mic); + gl.uniform1f(locations.voice, d.voice); + gl.uniform1f(locations.scale, d.scale); + gl.uniform2f(locations.tap, d.tapX, d.tapY); + gl.uniform1f(locations.tapTime, d.tapAge); + gl.uniform2f(locations.hover, d.hoverX, d.hoverY); + gl.uniform1f(locations.hoverAmt, d.hoverAmount); + gl.drawArrays(gl.TRIANGLES, 0, 3); + }; + + const dispose = () => { + canvas.removeEventListener('webglcontextlost', handleLost); + if (!gl || lost) return; + gl.deleteBuffer(buffer); + gl.deleteProgram(program); + gl.deleteShader(vertex); + gl.deleteShader(fragment); + }; + + return { dispose, draw }; +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/renderers/types.ts b/src/plugins/AIComponents/VoiceVisualization/renderers/types.ts new file mode 100644 index 000000000..51741d515 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/renderers/types.ts @@ -0,0 +1,20 @@ +import type { VoiceVisualizationDriver } from '../driver'; + +/** The surface a renderer draws on, in device pixels. */ +export type VoiceFrame = { + /** 1 to draw for a dark background, 0 for a light one. */ + dark: number; + /** Device pixels per CSS pixel. */ + dpr: number; + driver: VoiceVisualizationDriver; + height: number; + width: number; +}; + +export type VoiceRenderer = { + draw: (frame: VoiceFrame) => void; + dispose?: () => void; +}; + +export const rgba = (c: ArrayLike, alpha: number) => + `rgba(${Math.round(c[0] * 255)}, ${Math.round(c[1] * 255)}, ${Math.round(c[2] * 255)}, ${Math.min(Math.max(alpha, 0), 1)})`; diff --git a/src/plugins/AIComponents/VoiceVisualization/renderers/waveRenderer.ts b/src/plugins/AIComponents/VoiceVisualization/renderers/waveRenderer.ts new file mode 100644 index 000000000..1e9ee002d --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/renderers/waveRenderer.ts @@ -0,0 +1,123 @@ +import type { VoiceVisualizationDriver } from '../driver'; +import { clamp, mix, saturate, smoothstep, speechEnv, vivid } from '../math'; +import { rgba, type VoiceFrame, type VoiceRenderer } from './types'; + +const POINTS = 160; +const STOPS = 13; +const START = -0.42; +const END = 0.42; + +const sq = (x: number) => x * x; + +const bandCenter = (i: number, seg: number, t: number) => { + switch (i) { + case 0: + return 0.5 * seg + 0.12 * Math.sin(t * 0.13); + case 1: + return 1.5 * seg + 0.13 * Math.sin(t * 0.11 + 2.1); + case 2: + return 2.5 * seg + 0.12 * Math.sin(t * 0.17 + 4.2); + case 3: + return 3.5 * seg + 0.12 * Math.sin(t * 0.15 + 1); + default: + return 4.5 * seg + 0.12 * Math.sin(t * 0.1 + 3.3); + } +}; + +/** Whether a point, centred and normalized to the shorter side with y up, lands on the string. */ +export const waveContains = (x: number, y: number) => + Math.abs(x) < 0.5 && Math.abs(y) < 0.14; + +/** + * The wave: a string, flat at its ends, carrying a wave packet that erupts with speech, with up + * to five colors flowing along it in bands. Its buffers are reused from one frame to the next. + */ +export const createWaveRenderer = (ctx: CanvasRenderingContext2D): VoiceRenderer => { + const xs = new Float32Array(POINTS + 1); + const ys = new Float32Array(POINTS + 1); + const band = Array.from({ length: 5 }, () => [0, 0, 0]); + const color = [0, 0, 0]; + + /** The string's color at `x`: the bands weighed by how near their centres are, faded at the ends. */ + const stopColor = (d: VoiceVisualizationDriver, x: number) => { + const seg = 1 / Math.max(d.count, 1); + const bandWidth = seg * 0.7; + const gx = clamp(x + 0.5, 0, 1); + let total = 1e-4; + color.fill(0); + for (let i = 0; i < 5; i++) { + const present = i === 0 ? 1 : clamp(d.count - i, 0, 1); + const weight = + Math.exp(-sq((gx - bandCenter(i, seg, d.time)) / bandWidth)) * present; + for (let j = 0; j < 3; j++) color[j] += band[i][j] * weight; + total += weight; + } + for (let j = 0; j < 3; j++) color[j] /= total; + saturate(color, 1.2); + saturate(color, d.sat); + const alpha = smoothstep(0.4, 0.24, Math.abs(x)) * d.bright; + return rgba(color, alpha); + }; + + const draw = ({ dark, dpr, driver: d, height, width }: VoiceFrame) => { + ctx.clearRect(0, 0, width, height); + const m = Math.min(width, height); + if (m <= 0) return; + const t = d.time; + const react = d.react; + const center = d.load * 0.3 * Math.sin(t * 1.3); + const freq = 24 + 26 * react + 10 * d.flow; + const speak = smoothstep(0.6, 1, react); + const drive = mix(speak * 0.26 * speechEnv(t), 0.3 * d.mic, d.voice); + const reach = 0.018 + 0.022 * react + drive + 0.14 * d.flow + 0.1 * d.load; + const tapAlive = d.tapAge < 1.5; + const hovered = d.hoverAmount > 0.001; + + for (let i = 0; i <= POINTS; i++) { + const x = START + ((END - START) * i) / POINTS; + let y = Math.exp(-sq((x - center) / 0.18)) * reach * Math.sin(x * freq - t * 5); + if (tapAlive) { + y += + Math.exp(-sq((x - d.tapX) / 0.12)) * + Math.sin((x - d.tapX) * 40 - d.tapAge * 24) * + Math.exp(-d.tapAge * 2.6) * + 0.05; + } + if (hovered) { + y += + Math.exp(-sq((x - d.hoverX) / 0.12)) * + Math.sin((x - d.hoverX) * 40 - t * 7) * + d.hoverAmount * + 0.04; + } + xs[i] = width / 2 + x * m * d.scale; + ys[i] = height / 2 - y * m * d.scale; + } + + // Broad color bands drifting along the string, its ends fading out. + for (let i = 0; i < 5; i++) vivid(d.color(i), dark, band[i]); + const gradient = ctx.createLinearGradient( + width / 2 + START * m, + 0, + width / 2 + END * m, + 0, + ); + for (let s = 0; s < STOPS; s++) { + gradient.addColorStop( + s / (STOPS - 1), + stopColor(d, START + ((END - START) * s) / (STOPS - 1)), + ); + } + + ctx.beginPath(); + ctx.moveTo(xs[0], ys[0]); + for (let i = 1; i <= POINTS; i++) ctx.lineTo(xs[i], ys[i]); + ctx.strokeStyle = gradient; + ctx.lineWidth = Math.max(0.006 * m, 1.5 * dpr); + ctx.lineCap = 'round'; + ctx.lineJoin = 'round'; + ctx.stroke(); + }; + + return { draw }; +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/types.ts b/src/plugins/AIComponents/VoiceVisualization/types.ts new file mode 100644 index 000000000..eb64ea4d3 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/types.ts @@ -0,0 +1,55 @@ +import type { ComponentPropsWithoutRef } from 'react'; + +/** + * What a voice agent is doing, which a voice visualization shows as motion. + * + * The states differ only in how a visualization moves, never in its colors, and changing the + * state eases from one motion into the next. + * + * - `idle`: waiting to be spoken to, a calm, barely-there shimmer. + * - `connecting`: not ready yet, a loader sweep with a pulsing breath. + * - `listening`: hearing somebody, a gentle expand and contract. + * - `thinking`: working on an answer, a slow turn or a travelling wave. + * - `speaking`: talking, strong motion that ebbs and pauses like speech. + */ +export type VoiceAgentState = + | 'idle' + | 'connecting' + | 'listening' + | 'thinking' + | 'speaking'; + +/** + * How `VoiceGlow` is drawn. Either way it moves the same; only the look and the cost differ. + * + * - `auto`: WebGL where the browser can create a context, otherwise Canvas. + * - `webgl`: a shader computing every pixel, the richest look and the most GPU. Canvas when + * WebGL is unavailable or its context is lost. + * - `canvas`: a handful of 2D gradients, cheaper, without the shader's marbling or its ripples + * from a tap or a hovering pointer. + */ +export type VoiceGlowRendering = 'auto' | 'webgl' | 'canvas'; + +export type VoiceVisualizationProps = Omit< + ComponentPropsWithoutRef<'div'>, + 'children' +> & { + /** What the agent is doing, which sets how the visualization moves. */ + state: VoiceAgentState; + /** + * The level of whoever is talking, from 0 to 1, read once a frame. Feed it at 15 Hz or more + * for the visualization to move with the words; returning `null` animates speech on its own. + * See `useAudioLevel` to read it from a `MediaStream`. + */ + audioLevel?: () => number | null | undefined; + /** + * One to five CSS colors. Past five are ignored. When omitted or empty, the theme's accent + * color (`--str-chat__ai-voice-color`) is used. + */ + colors?: string[]; + /** From 0 to 2: how far and how fast it moves. Colors never change with it. Defaults to 1. */ + expressivity?: number; +}; + +/** A color as hue (a fraction of a turn), saturation and value, each from 0 to 1. */ +export type Hsv = { hue: number; saturation: number; value: number }; diff --git a/src/plugins/AIComponents/VoiceVisualization/useAudioLevel.ts b/src/plugins/AIComponents/VoiceVisualization/useAudioLevel.ts new file mode 100644 index 000000000..cf193b94d --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/useAudioLevel.ts @@ -0,0 +1,79 @@ +import { useCallback, useEffect, useRef } from 'react'; + +/** Below this RMS the signal is treated as room noise. */ +const NOISE_FLOOR = 0.01; +/** Maps the RMS of speech at a normal distance (about 0.05 to 0.2) across most of 0 to 1. */ +const GAIN = 5; + +type AudioContextConstructor = new () => AudioContext; + +const getAudioContext = (): AudioContextConstructor | undefined => { + if (typeof window === 'undefined') return undefined; + return ( + window.AudioContext ?? + (window as Window & { webkitAudioContext?: AudioContextConstructor }) + .webkitAudioContext + ); +}; + +/** The RMS of a block of 8-bit time-domain samples centred on 128, from 0 to 1. */ +export const rootMeanSquare = (samples: Uint8Array) => { + if (!samples.length) return 0; + let sum = 0; + for (let i = 0; i < samples.length; i++) { + const value = (samples[i] - 128) / 128; + sum += value * value; + } + return Math.sqrt(sum / samples.length); +}; + +/** An RMS as a level from 0 to 1, with room noise cut and speech spread across the range. */ +export const levelFromRms = (rms: number) => + Math.min(Math.max((rms - NOISE_FLOOR) * GAIN, 0), 1); + +/** + * Reads the live level of a `MediaStream`'s audio, such as a microphone from + * `getUserMedia({ audio: true })` or a remote participant's track, for `VoiceWave` and + * `VoiceGlow`'s `audioLevel`. + * + * Returns a stable getter that measures the level when called, from 0 to 1, so a visualization + * reading it once a frame re-renders nothing. It returns `null` while there is no stream or the + * browser has no Web Audio, which lets the visualization animate speech on its own. + * + * The audio graph is torn down when the stream changes and on unmount; the stream's tracks are + * left running, since they belong to the caller. + */ +export const useAudioLevel = (stream: MediaStream | null | undefined) => { + const analyserRef = useRef<{ + analyser: AnalyserNode; + samples: Uint8Array; + } | null>(null); + + useEffect(() => { + const AudioContextClass = getAudioContext(); + if (!stream || !AudioContextClass || !stream.getAudioTracks().length) return; + + const context = new AudioContextClass(); + const source = context.createMediaStreamSource(stream); + const analyser = context.createAnalyser(); + analyser.fftSize = 256; + source.connect(analyser); + analyserRef.current = { analyser, samples: new Uint8Array(analyser.fftSize) }; + // A context made outside a user gesture can start suspended. + if (context.state === 'suspended') context.resume?.().catch(() => undefined); + + return () => { + analyserRef.current = null; + source.disconnect(); + analyser.disconnect(); + if (context.state !== 'closed') context.close().catch(() => undefined); + }; + }, [stream]); + + return useCallback((): number | null => { + const current = analyserRef.current; + if (!current) return null; + current.analyser.getByteTimeDomainData(current.samples); + return levelFromRms(rootMeanSquare(current.samples)); + }, []); +}; diff --git a/src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx b/src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx new file mode 100644 index 000000000..261532a93 --- /dev/null +++ b/src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx @@ -0,0 +1,259 @@ +import React, { type ReactNode } from 'react'; +import { act, fireEvent, render, screen } from '@testing-library/react'; + +import { VoiceGlow, VoiceWave } from '..'; +import { TranslationProvider } from '../../../context'; +import en from '../../../i18n/en.json'; + +type Recorded = { args: unknown[]; name: string }; + +/** A 2D context that records every call and accepts any property. */ +const createFakeContext = (calls: Recorded[]) => + new Proxy({} as Record, { + get(target, name) { + if (name in target) return target[name]; + const fn = (...args: unknown[]) => { + calls.push({ args, name: String(name) }); + if (name === 'createLinearGradient' || name === 'createRadialGradient') { + return { + addColorStop: (offset: number, color: string) => + calls.push({ args: [offset, color], name: 'addColorStop' }), + }; + } + return undefined; + }; + target[name] = fn; + return fn; + }, + }); + +/** English, with interpolation, as Streami18n would give. */ +const enT = ((key: string, options?: Record) => + ((en as Record)[key] || key).replace( + /\{\{\s*(\w+)\s*\}\}/g, + (_, name) => options?.[name] ?? '', + )) as never; + +const WithEnglish = ({ children }: { children: ReactNode }) => ( + null) as never, userLanguage: 'en' }} + > + {children} + +); + +describe('VoiceWave and VoiceGlow', () => { + let frames: FrameRequestCallback[]; + let now: number; + let calls: Recorded[]; + let contextTypes: string[]; + + const step = (count = 1) => { + for (let i = 0; i < count; i++) { + const pending = frames; + frames = []; + now += 1000 / 60; + act(() => pending.forEach((callback) => callback(now))); + } + }; + + beforeEach(() => { + frames = []; + now = 1000; + calls = []; + contextTypes = []; + vi.spyOn(window, 'requestAnimationFrame').mockImplementation((callback) => { + frames.push(callback); + return frames.length; + }); + vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => undefined); + // jsdom lays nothing out; the renderers draw nothing in a box with no size. + vi.spyOn(HTMLElement.prototype, 'clientWidth', 'get').mockReturnValue(200); + vi.spyOn(HTMLElement.prototype, 'clientHeight', 'get').mockReturnValue(200); + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockImplementation(function ( + this: HTMLCanvasElement, + type: string, + ) { + contextTypes.push(type); + return type === '2d' ? (createFakeContext(calls) as never) : null; + } as never); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('names the visualization and its state, and follows a state change', () => { + const { rerender } = render( + + + , + ); + expect(screen.getByRole('img', { name: 'Voice agent: Ready' })).toBeInTheDocument(); + + rerender( + + + , + ); + expect(screen.getByRole('img', { name: 'Voice agent: Speaking' })).toHaveAttribute( + 'data-state', + 'speaking', + ); + }); + + it('renders a hidden canvas inside its root classes and passes div props through', () => { + const { container } = render( + <> + + + , + ); + const wave = screen.getByTestId('wave'); + expect(wave).toHaveClass( + 'str-chat__ai-voice-visualization', + 'str-chat__ai-voice-wave', + 'custom', + ); + expect(wave.querySelector('canvas')).toHaveAttribute('aria-hidden', 'true'); + const glow = container.querySelector('.str-chat__ai-voice-glow') as HTMLElement; + expect(glow.style.width).toBe('100px'); + }); + + it('reads the audio level once a frame', () => { + const audioLevel = vi.fn(() => 0.5); + render(); + step(); + audioLevel.mockClear(); + + step(3); + + expect(audioLevel).toHaveBeenCalledTimes(3); + }); + + it('keeps animating with an audio level that is not a number', () => { + render( Number.NaN} state='speaking' />); + step(3); + expect(calls.filter((call) => call.name === 'clearRect').length).toBe(3); + }); + + it('draws the theme color when no colors are given, and the colors when they are', () => { + const { rerender } = render( + , + ); + step(); + const stops = () => + calls.filter((call) => call.name === 'addColorStop').map((call) => call.args[1]); + expect(stops().some((color) => String(color).startsWith('rgba(255, 0, 0,'))).toBe( + true, + ); + + calls.length = 0; + rerender( + , + ); + // The palette eases toward the new color. + step(120); + calls.length = 0; + step(); + expect(stops().some((color) => String(color).startsWith('rgba(0, 255, 0,'))).toBe( + true, + ); + }); + + it('bounces on a tap inside the globe and still lets a parent handle the click', () => { + const onClick = vi.fn(); + const onPointerUp = vi.fn(); + render( +
+ +
, + ); + const glow = screen.getByRole('img'); + vi.spyOn(glow, 'getBoundingClientRect').mockReturnValue({ + bottom: 200, + height: 200, + left: 0, + right: 200, + top: 0, + width: 200, + x: 0, + y: 0, + } as DOMRect); + step(); + + fireEvent.pointerUp(glow, { clientX: 100, clientY: 100 }); + fireEvent.click(glow); + calls.length = 0; + step(3); + + expect(onPointerUp).toHaveBeenCalledTimes(1); + expect(onClick).toHaveBeenCalledTimes(1); + const scales = calls + .filter((call) => call.name === 'scale') + .map((call) => call.args[0] as number); + expect(Math.min(...scales)).toBeLessThan(1); + }); + + it('ignores a tap outside the globe', () => { + render(); + const glow = screen.getByRole('img'); + vi.spyOn(glow, 'getBoundingClientRect').mockReturnValue({ + height: 200, + left: 0, + top: 0, + width: 200, + } as DOMRect); + step(); + + fireEvent.pointerUp(glow, { clientX: 2, clientY: 2 }); + calls.length = 0; + step(3); + + const scales = calls + .filter((call) => call.name === 'scale') + .map((call) => call.args[0]); + expect(scales.every((scale) => scale === 1)).toBe(true); + }); + + it('falls back to Canvas when WebGL is unavailable', () => { + render(); + + expect(contextTypes).toEqual(['webgl', '2d']); + expect(screen.getByRole('img')).toHaveAttribute('data-rendering', 'canvas'); + step(); + expect(calls.some((call) => call.name === 'createRadialGradient')).toBe(true); + }); + + it('never asks for WebGL when told to draw on Canvas', () => { + render(); + + expect(contextTypes).toEqual(['2d']); + }); + + it('renders its labelled root even where no canvas context exists', () => { + vi.mocked(HTMLCanvasElement.prototype.getContext).mockReturnValue(null); + + render(); + render(); + + expect(screen.getAllByRole('img')).toHaveLength(2); + expect(frames).toHaveLength(0); + }); + + it('stops the loop on unmount', () => { + const { unmount } = render(); + step(); + + unmount(); + + expect(window.cancelAnimationFrame).toHaveBeenCalled(); + }); +}); diff --git a/src/plugins/AIComponents/__tests__/a11y.test.tsx b/src/plugins/AIComponents/__tests__/a11y.test.tsx index 80dc76edf..56e232aa2 100644 --- a/src/plugins/AIComponents/__tests__/a11y.test.tsx +++ b/src/plugins/AIComponents/__tests__/a11y.test.tsx @@ -1,7 +1,7 @@ import React, { type ReactNode } from 'react'; import { render, screen } from '@testing-library/react'; -import { AIMessageComposer, AIStateIndicator } from '..'; +import { AIMessageComposer, AIStateIndicator, VoiceGlow, VoiceWave } from '..'; import { Item } from '../AIMessageComposer/AttachmentPreview'; import { TranslationProvider } from '../../../context'; import en from '../../../i18n/en.json'; @@ -90,6 +90,33 @@ describe('AI components accessibility', () => { expect(screen.getByRole('button', { name: 'Retry upload' })).toBeInTheDocument(); }); + it('voice visualizations have no axe violations and are named with their state', async () => { + const interpolatingT = ((key: string, options?: Record) => + ((en as Record)[key] || key).replace( + /\{\{\s*(\w+)\s*\}\}/g, + (_, name) => options?.[name] ?? '', + )) as never; + const { container } = render( + null) as never, + userLanguage: 'en', + }} + > + + + , + ); + expect(await axe(container)).toHaveNoViolations(); + expect( + screen.getByRole('img', { name: 'Voice agent: Listening' }), + ).toBeInTheDocument(); + expect( + screen.getByRole('img', { name: 'Voice agent: Thinking' }), + ).toBeInTheDocument(); + }); + it('announces the AI state politely', () => { render(); const status = screen.getByRole('status'); diff --git a/src/plugins/AIComponents/__tests__/classNames.test.tsx b/src/plugins/AIComponents/__tests__/classNames.test.tsx index 1f296fdf8..8d84650f7 100644 --- a/src/plugins/AIComponents/__tests__/classNames.test.tsx +++ b/src/plugins/AIComponents/__tests__/classNames.test.tsx @@ -1,7 +1,14 @@ import React from 'react'; import { render } from '@testing-library/react'; -import { AIMarkdown, AIMessageComposer, AIStateIndicator, StreamingMessage } from '..'; +import { + AIMarkdown, + AIMessageComposer, + AIStateIndicator, + StreamingMessage, + VoiceGlow, + VoiceWave, +} from '..'; it('emits no legacy aicr__ classes', () => { const { container } = render( @@ -13,10 +20,14 @@ it('emits no legacy aicr__ classes', () => { + + , ); expect(container.innerHTML).not.toContain('aicr__'); expect( container.querySelector('.str-chat__ai-message-composer__form'), ).toBeInTheDocument(); + expect(container.querySelector('.str-chat__ai-voice-wave')).toBeInTheDocument(); + expect(container.querySelector('.str-chat__ai-voice-glow')).toBeInTheDocument(); }); diff --git a/src/plugins/AIComponents/__tests__/hydration.test.tsx b/src/plugins/AIComponents/__tests__/hydration.test.tsx index 7d9f4c046..d41323d5d 100644 --- a/src/plugins/AIComponents/__tests__/hydration.test.tsx +++ b/src/plugins/AIComponents/__tests__/hydration.test.tsx @@ -3,7 +3,7 @@ import { act } from '@testing-library/react'; import { renderToString } from 'react-dom/server'; import { hydrateRoot } from 'react-dom/client'; -import { AIMessageComposer } from '..'; +import { AIMessageComposer, VoiceGlow, VoiceWave } from '..'; import { installFakeSpeechRecognition, uninstallFakeSpeechRecognition, @@ -48,3 +48,38 @@ describe('AIMessageComposer hydration', () => { container.remove(); }); }); + +describe('VoiceWave and VoiceGlow hydration', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('hydrates server markup without mismatches', () => { + const VoiceTree = () => ( + <> + + + + ); + const html = renderToString(); + const container = document.createElement('div'); + container.innerHTML = html; + document.body.appendChild(container); + const serverCanvases = Array.from(container.querySelectorAll('canvas')); + + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined); + const onRecoverableError = vi.fn(); + act(() => { + hydrateRoot(container, , { onRecoverableError }); + }); + + expect(onRecoverableError).not.toHaveBeenCalled(); + expect(consoleError).not.toHaveBeenCalled(); + // The wave keeps its server canvas; the glow may swap its canvas for a Canvas fallback. + expect(container.querySelector('.str-chat__ai-voice-wave canvas')).toBe( + serverCanvases[0], + ); + expect(container.querySelectorAll('[role="img"]')).toHaveLength(2); + container.remove(); + }); +}); diff --git a/src/plugins/AIComponents/__tests__/i18n.test.tsx b/src/plugins/AIComponents/__tests__/i18n.test.tsx index 740f15044..2a556d23b 100644 --- a/src/plugins/AIComponents/__tests__/i18n.test.tsx +++ b/src/plugins/AIComponents/__tests__/i18n.test.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { render, screen } from '@testing-library/react'; -import { AIMessageComposer, AIStateIndicator } from '..'; +import { AIMessageComposer, AIStateIndicator, VoiceGlow, VoiceWave } from '..'; import { TranslationProvider } from '../../../context'; import en from '../../../i18n/en.json'; import { @@ -43,6 +43,31 @@ describe('AI components i18n', () => { expect(container.textContent).toMatch(/^T:/); }); + it('translates the voice visualization names and states', () => { + const translate = vi.fn( + (key: string, options?: { state?: string }) => + `T:${key}${options?.state ? `|${options.state}` : ''}`, + ); + render( + null) as never, + userLanguage: 'en', + }} + > + + + , + ); + expect( + screen.getByRole('img', { name: 'T:aria/Voice agent: {{ state }}|T:Speaking' }), + ).toBeInTheDocument(); + expect( + screen.getByRole('img', { name: 'T:aria/Voice agent: {{ state }}|T:Ready' }), + ).toBeInTheDocument(); + }); + it('renders real English values for aria keys (no aria/ prefix)', () => { const enT = ((key: string) => (en as Record)[key] ?? key) as never; render( diff --git a/src/plugins/AIComponents/__tests__/ssr.test.tsx b/src/plugins/AIComponents/__tests__/ssr.test.tsx index 61d27548f..7360cb583 100644 --- a/src/plugins/AIComponents/__tests__/ssr.test.tsx +++ b/src/plugins/AIComponents/__tests__/ssr.test.tsx @@ -2,7 +2,14 @@ import React from 'react'; import { renderToString } from 'react-dom/server'; -import { AIMarkdown, AIMessageComposer, AIStateIndicator, StreamingMessage } from '..'; +import { + AIMarkdown, + AIMessageComposer, + AIStateIndicator, + StreamingMessage, + VoiceGlow, + VoiceWave, +} from '..'; it('server-renders without touching browser globals', () => { expect(typeof window).toBe('undefined'); @@ -15,10 +22,20 @@ it('server-renders without touching browser globals', () => { + + , ); expect(html).toContain('bold'); expect(html).toContain('streamed-ssr-marker'); expect(html).toContain('Thinking'); expect(html).toContain(' { expect(css).toContain( 'var(--str-chat__ai-table-header-bg, var(--str-chat__background-core-surface-subtle, #f8f9fa))', ); + expect(css).toContain( + 'var(--str-chat__ai-voice-color, var(--str-chat__accent-primary, #005fff))', + ); + expect(css).toContain( + 'var(--str-chat__ai-voice-backdrop, var(--str-chat__background-core-elevation-0, #ffffff))', + ); }); it('uses no colour literal outside a var() fallback (so SCR dark mode applies)', () => { @@ -103,6 +109,12 @@ describe('AI components stylesheet accessibility', () => { ); }); + it('restores a visible focus indicator on a focusable voice visualization', () => { + expect(css).toMatch( + /\.str-chat__ai-voice-visualization:focus-visible[^{]*\{[^}]*outline:\s*2px solid var\(--str-chat__border-utility-focused/, + ); + }); + it('disables the state indicator animations for reduced motion', () => { const reduced = css.split('@media (prefers-reduced-motion: reduce)')[1] ?? ''; expect(reduced).toMatch( diff --git a/src/plugins/AIComponents/__tests__/useAudioLevel.test.ts b/src/plugins/AIComponents/__tests__/useAudioLevel.test.ts new file mode 100644 index 000000000..b48ed369d --- /dev/null +++ b/src/plugins/AIComponents/__tests__/useAudioLevel.test.ts @@ -0,0 +1,110 @@ +import { renderHook } from '@testing-library/react'; + +import { useAudioLevel } from '..'; +import { levelFromRms, rootMeanSquare } from '../VoiceVisualization/useAudioLevel'; +import { AudioContextMock } from '../../../mock-builders/browser/AudioContext'; + +let samples = new Uint8Array(0); + +class AnalyserFake { + fftSize = 2048; + connect = vi.fn(); + disconnect = vi.fn(); + getByteTimeDomainData = vi.fn((out: Uint8Array) => + out.set(samples.subarray(0, out.length)), + ); +} + +class AudioContextFake extends AudioContextMock { + static instances: AudioContextFake[] = []; + state = 'running'; + analyser = new AnalyserFake(); + source = { connect: vi.fn(), disconnect: vi.fn() }; + createAnalyser = vi.fn(() => this.analyser) as never; + createMediaStreamSource = vi.fn(() => this.source) as never; + close = vi.fn(() => { + this.state = 'closed'; + return Promise.resolve(); + }) as never; + constructor() { + super(); + AudioContextFake.instances.push(this); + } +} + +const stream = (audioTracks = 1) => + ({ getAudioTracks: () => Array.from({ length: audioTracks }, () => ({})) }) as never; + +describe('useAudioLevel', () => { + beforeEach(() => { + AudioContextFake.instances = []; + samples = new Uint8Array(256).fill(128); + vi.stubGlobal('AudioContext', AudioContextFake); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('measures the RMS of time-domain samples', () => { + expect(rootMeanSquare(new Uint8Array([128, 128]))).toBe(0); + expect(rootMeanSquare(new Uint8Array([0, 0]))).toBe(1); + expect(rootMeanSquare(new Uint8Array([192, 64]))).toBeCloseTo(0.5, 5); + expect(rootMeanSquare(new Uint8Array(0))).toBe(0); + }); + + it('cuts room noise and spreads speech across 0 to 1', () => { + expect(levelFromRms(0.005)).toBe(0); + expect(levelFromRms(0.11)).toBeCloseTo(0.5, 5); + expect(levelFromRms(1)).toBe(1); + }); + + it('returns null without a stream', () => { + const { result } = renderHook(() => useAudioLevel(null)); + + expect(result.current()).toBeNull(); + expect(AudioContextFake.instances).toHaveLength(0); + }); + + it('returns null for a stream with no audio', () => { + const { result } = renderHook(() => useAudioLevel(stream(0))); + + expect(result.current()).toBeNull(); + }); + + it('reads the level of a stream when asked', () => { + const { result } = renderHook(() => useAudioLevel(stream())); + const [context] = AudioContextFake.instances; + expect(context.source.connect).toHaveBeenCalledWith(context.analyser); + expect(result.current()).toBe(0); + + samples = new Uint8Array(256).map((_, i) => (i % 2 ? 192 : 64)); + + expect(result.current()).toBe(1); + expect(context.analyser.getByteTimeDomainData).toHaveBeenCalledTimes(2); + }); + + it('keeps the same getter across renders', () => { + const { rerender, result } = renderHook(() => useAudioLevel(stream())); + const first = result.current; + + rerender(); + + expect(result.current).toBe(first); + }); + + it('closes the audio graph when the stream changes and on unmount', () => { + const { rerender, unmount } = renderHook(({ s }) => useAudioLevel(s), { + initialProps: { s: stream() }, + }); + const [first] = AudioContextFake.instances; + + rerender({ s: stream() }); + expect(first.close).toHaveBeenCalled(); + expect(first.source.disconnect).toHaveBeenCalled(); + + const second = AudioContextFake.instances[1]; + unmount(); + expect(second.close).toHaveBeenCalled(); + }); +}); diff --git a/src/plugins/AIComponents/__tests__/voiceDriver.test.ts b/src/plugins/AIComponents/__tests__/voiceDriver.test.ts new file mode 100644 index 000000000..d86a848b2 --- /dev/null +++ b/src/plugins/AIComponents/__tests__/voiceDriver.test.ts @@ -0,0 +1,238 @@ +import { easeOut, scaleAt, VoiceVisualizationBounce } from '../VoiceVisualization/bounce'; +import { + ease, + NO_TAP, + type VoiceDriverInput, + VoiceVisualizationDriver, +} from '../VoiceVisualization/driver'; +import type { Hsv, VoiceAgentState } from '../VoiceVisualization/types'; + +// Ported from stream-chat-android's VoiceVisualizationDriverTest and VoiceVisualizationBounceTest, +// so both SDKs move alike. + +const blue: Hsv = { hue: 218 / 360, saturation: 1, value: 1 }; +const red: Hsv = { hue: 0, saturation: 1, value: 1 }; +const frame = 1 / 60; + +const input = ( + state: VoiceAgentState, + overrides: Partial> = {}, +): VoiceDriverInput => ({ + audioLevel: null, + colors: [blue], + expressivity: 1, + reduceMotion: false, + state, + ...overrides, +}); + +const run = (driver: VoiceVisualizationDriver, seconds: number) => { + for (let i = 0; i < Math.floor(seconds / frame); i++) driver.advance(frame); +}; + +describe('VoiceVisualizationDriver', () => { + it('starts settled on the state', () => { + const driver = new VoiceVisualizationDriver(input('speaking')); + + expect(driver.react).toBe(1); + expect(driver.load).toBe(0); + expect(driver.flow).toBe(0); + expect(driver.bright).toBe(1); + expect(driver.time).toBe(0); + expect(driver.count).toBe(1); + expect(driver.scale).toBe(1); + expect(driver.tapAge).toBe(NO_TAP); + expect(driver.color(0)).toEqual(blue); + expect(driver.color(4)).toEqual(blue); + }); + + it('eases toward the motion of a new state', () => { + const driver = new VoiceVisualizationDriver(input('idle')); + driver.input = input('connecting'); + + driver.advance(frame); + expect(driver.load).toBeCloseTo(0.08, 4); + run(driver, 2); + expect(driver.load).toBeCloseTo(1, 3); + expect(driver.react).toBeCloseTo(0, 3); + }); + + it('eases the same at any frame rate', () => { + const at60 = new VoiceVisualizationDriver(input('idle')); + const at120 = new VoiceVisualizationDriver(input('idle')); + at60.input = input('speaking'); + at120.input = input('speaking'); + + for (let i = 0; i < 30; i++) at60.advance(1 / 60); + for (let i = 0; i < 60; i++) at120.advance(1 / 120); + + expect(at60.react).toBeCloseTo(at120.react, 4); + expect(Math.abs(at60.time - at120.time)).toBeLessThan(1e-2); + }); + + it('clamps a long frame', () => { + const driver = new VoiceVisualizationDriver(input('idle')); + + driver.advance(5); + + expect(driver.time).toBeCloseTo(0.05 * 0.4, 6); + }); + + it('turns the flow only while thinking', () => { + const driver = new VoiceVisualizationDriver(input('speaking')); + run(driver, 1); + expect(driver.flowSpin).toBe(0); + + driver.input = input('thinking'); + run(driver, 1); + expect(driver.flowSpin).toBeGreaterThan(0); + }); + + it('keeps the colors turning in every state', () => { + const driver = new VoiceVisualizationDriver(input('idle')); + + run(driver, 1); + + expect(driver.orbSpin).toBeGreaterThan(0); + }); + + it('scales amplitude and speed with expressivity', () => { + const lively = new VoiceVisualizationDriver(input('speaking', { expressivity: 2 })); + const still = new VoiceVisualizationDriver(input('speaking', { expressivity: 0 })); + + expect(lively.react).toBeCloseTo(1.85, 5); + expect(still.react).toBeCloseTo(0.15, 5); + lively.advance(0.05); + expect(lively.time).toBeCloseTo(0.05 * 1.25 * 1.4, 5); + }); + + it('slows time and ignores the pointer under reduced motion', () => { + const driver = new VoiceVisualizationDriver(input('idle', { reduceMotion: true })); + + driver.tap(0.1, 0.1); + driver.hover(0.1, 0.1); + driver.advance(0.05); + + expect(driver.time).toBeCloseTo(0.05 * 0.07 * 0.4, 7); + expect(driver.tapAge).toBeGreaterThan(NO_TAP); + expect(driver.scale).toBe(1); + expect(driver.hoverAmount).toBe(0); + }); + + it('stops hovering when reduced motion turns on', () => { + const driver = new VoiceVisualizationDriver(input('idle')); + driver.hover(0.1, 0.1); + run(driver, 1); + expect(driver.hoverAmount).toBeGreaterThan(0.9); + + driver.input = input('idle', { reduceMotion: true }); + run(driver, 1); + expect(driver.hoverAmount).toBeCloseTo(0, 3); + }); + + it('ripples from a tap and bounces', () => { + const driver = new VoiceVisualizationDriver(input('idle')); + + driver.tap(0.1, -0.2); + expect(driver.tapX).toBe(0.1); + expect(driver.tapY).toBe(-0.2); + expect(driver.tapAge).toBe(0); + + driver.advance(0.05); + expect(driver.tapAge).toBeCloseTo(0.05, 6); + expect(driver.scale).toBeLessThan(1); + run(driver, 0.5); + expect(driver.scale).toBe(1); + }); + + it('eases hovering in and out', () => { + const driver = new VoiceVisualizationDriver(input('idle')); + + driver.hover(0.2, 0.1); + run(driver, 1); + expect(driver.hoverAmount).toBeCloseTo(1, 3); + expect(driver.hoverX).toBe(0.2); + + driver.hover(null, null); + run(driver, 1); + expect(driver.hoverAmount).toBeCloseTo(0, 3); + expect(driver.hoverX).toBe(0.2); + }); + + it('lifts the motion to a live audio level', () => { + const driver = new VoiceVisualizationDriver(input('idle', { audioLevel: 0.8 })); + + run(driver, 2); + expect(driver.mic).toBeCloseTo(0.8, 3); + expect(driver.voice).toBeCloseTo(1, 3); + expect(driver.react).toBeCloseTo(0.8, 3); + + driver.input = input('idle'); + run(driver, 2); + expect(driver.voice).toBeCloseTo(0, 3); + expect(driver.react).toBeCloseTo(0.12, 3); + }); + + it('eases a hue the short way round', () => { + const driver = new VoiceVisualizationDriver( + input('idle', { colors: [{ hue: 0.95, saturation: 1, value: 1 }] }), + ); + driver.input = input('idle', { colors: [{ hue: 0.05, saturation: 1, value: 1 }] }); + + driver.advance(frame); + expect(driver.color(0).hue).toBeCloseTo(0.962, 4); + run(driver, 2); + expect(driver.color(0).hue).toBeCloseTo(0.05, 3); + }); + + it('repeats the first color in empty slots and crossfades the count', () => { + const driver = new VoiceVisualizationDriver(input('idle', { colors: [blue, red] })); + expect(driver.color(1)).toEqual(red); + expect(driver.color(2)).toEqual(blue); + + driver.input = input('idle', { colors: [blue, red, red] }); + driver.advance(frame); + expect(driver.count).toBeCloseTo(2.12, 4); + }); + + it('moves the per-frame share at 60 Hz', () => { + expect(ease(0.08, 1 / 60)).toBeCloseTo(0.08, 5); + expect(ease(0.08, 0)).toBe(0); + }); +}); + +describe('VoiceVisualizationBounce', () => { + it('follows the keyframes', () => { + expect(scaleAt(0)).toBe(1); + expect(scaleAt(0.15)).toBeCloseTo(0.965, 5); + expect(scaleAt(0.3)).toBeCloseTo(0.93, 5); + expect(scaleAt(0.62)).toBeCloseTo(1.03, 5); + expect(scaleAt(1)).toBeCloseTo(1, 5); + }); + + it('times the run on the CSS ease-out', () => { + expect(easeOut(0)).toBeCloseTo(0, 4); + expect(Math.abs(easeOut(0.5) - 0.684)).toBeLessThan(2e-3); + expect(easeOut(1)).toBeCloseTo(1, 4); + }); + + it('dips, overshoots and settles', () => { + const bounce = new VoiceVisualizationBounce(); + expect(bounce.scale).toBe(1); + + bounce.start(); + const scales = Array.from({ length: 21 }, () => { + const scale = bounce.scale; + bounce.advance(1 / 60); + return scale; + }); + const min = Math.min(...scales); + const max = Math.max(...scales); + + expect(scales[0]).toBeCloseTo(1, 5); + expect(Math.abs(min - 0.93)).toBeLessThan(0.01); + expect(Math.abs(max - 1.03)).toBeLessThan(0.01); + expect(scales.indexOf(min)).toBeLessThan(scales.indexOf(max)); + expect(bounce.scale).toBe(1); + }); +}); diff --git a/src/plugins/AIComponents/__tests__/voicePalette.test.ts b/src/plugins/AIComponents/__tests__/voicePalette.test.ts new file mode 100644 index 000000000..399a169ae --- /dev/null +++ b/src/plugins/AIComponents/__tests__/voicePalette.test.ts @@ -0,0 +1,83 @@ +import { audioLevelInput, expressivityInput } from '../VoiceVisualization/math'; +import { + luminance, + parseColor, + rgbToHsv, + toHsvPalette, +} from '../VoiceVisualization/palette'; + +describe('voice visualization palette', () => { + it('keeps a color’s full hue, saturation and value', () => { + const hsv = rgbToHsv([0, 95 / 255, 1]); + + expect(hsv.hue * 360).toBeCloseTo(217.6, 1); + expect(hsv.saturation).toBeCloseTo(1, 4); + expect(hsv.value).toBeCloseTo(1, 4); + }); + + it('keeps a muted color muted', () => { + // hsv(108°, 25%, 60%) + const hsv = rgbToHsv([0.48, 0.6, 0.45]); + + expect(hsv.hue).toBeCloseTo(0.3, 2); + expect(hsv.saturation).toBeCloseTo(0.25, 2); + expect(hsv.value).toBeCloseTo(0.6, 4); + }); + + it('gives a gray no saturation', () => { + const hsv = rgbToHsv([0.5, 0.5, 0.5]); + + expect(hsv.saturation).toBe(0); + expect(hsv.value).toBe(0.5); + }); + + it.each([ + ['#005fff', [0, 95 / 255, 1]], + ['#0af', [0, 170 / 255, 1]], + ['#005fff80', [0, 95 / 255, 1]], + ['rgb(0, 95, 255)', [0, 95 / 255, 1]], + ['rgba(0 95 255 / 0.5)', [0, 95 / 255, 1]], + ['rgb(0%, 50%, 100%)', [0, 0.5, 1]], + ])('reads %s', (color, expected) => { + const rgb = parseColor(color); + expect(rgb).not.toBeNull(); + rgb?.forEach((channel, i) => expect(channel).toBeCloseTo(expected[i], 4)); + }); + + it('skips colors it cannot read', () => { + expect(parseColor('not-a-color')).toBeNull(); + expect(parseColor('#12345')).toBeNull(); + expect(toHsvPalette(['nope', '#ff0000'])).toHaveLength(1); + }); + + it('keeps only the first five colors', () => { + expect(toHsvPalette(Array.from({ length: 7 }, () => '#ff0000'))).toHaveLength(5); + }); + + it('measures luminance for the light or dark look', () => { + expect(luminance('#ffffff')).toBeCloseTo(1, 4); + expect(luminance('#000000')).toBe(0); + expect(luminance('#1c1e22')).toBeLessThan(0.5); + expect(luminance('')).toBeNull(); + }); +}); + +describe('voice visualization inputs', () => { + it('clamps expressivity to 0-2 and replaces one that is not a number', () => { + expect(expressivityInput(3)).toBe(2); + expect(expressivityInput(-1)).toBe(0); + expect(expressivityInput(0.5)).toBe(0.5); + expect(expressivityInput(Number.NaN)).toBe(1); + expect(expressivityInput(Number.POSITIVE_INFINITY)).toBe(1); + expect(expressivityInput(undefined)).toBe(1); + }); + + it('clamps the audio level to 0-1 and treats one that is not a number as no audio', () => { + expect(audioLevelInput(2)).toBe(1); + expect(audioLevelInput(-1)).toBe(0); + expect(audioLevelInput(0.4)).toBe(0.4); + expect(audioLevelInput(Number.NaN)).toBeNull(); + expect(audioLevelInput(null)).toBeNull(); + expect(audioLevelInput(undefined)).toBeNull(); + }); +}); diff --git a/src/plugins/AIComponents/index.ts b/src/plugins/AIComponents/index.ts index 9a5cc1d2c..d9719b8be 100644 --- a/src/plugins/AIComponents/index.ts +++ b/src/plugins/AIComponents/index.ts @@ -2,3 +2,4 @@ export * from './AIMarkdown'; export * from './AIMessageComposer'; export * from './AIStateIndicator/AIStateIndicator'; export * from './StreamingMessage/StreamingMessage'; +export * from './VoiceVisualization'; diff --git a/src/plugins/AIComponents/styling/_tokens.scss b/src/plugins/AIComponents/styling/_tokens.scss index ee2a388bf..259776faa 100644 --- a/src/plugins/AIComponents/styling/_tokens.scss +++ b/src/plugins/AIComponents/styling/_tokens.scss @@ -122,6 +122,14 @@ $ai-tokens: ( null, hsla(220, 14%, 30%, 0.15), ), + 'voice-color': ( + '--str-chat__accent-primary', + #005fff, + ), + 'voice-backdrop': ( + '--str-chat__background-core-elevation-0', + #ffffff, + ), ); @function ai-token($name) { diff --git a/src/plugins/AIComponents/styling/index.scss b/src/plugins/AIComponents/styling/index.scss index 80ac8ba9e..f5d7e8cb2 100644 --- a/src/plugins/AIComponents/styling/index.scss +++ b/src/plugins/AIComponents/styling/index.scss @@ -2,5 +2,6 @@ @use './streaming-message.scss'; @use './ai-message-composer.scss'; @use './ai-state-indicator.scss'; +@use './voice-visualization.scss'; // Intentionally declares no `--str-chat__ai-*` custom properties; see _tokens.scss. diff --git a/src/plugins/AIComponents/styling/voice-visualization.scss b/src/plugins/AIComponents/styling/voice-visualization.scss new file mode 100644 index 000000000..cc1b9b82b --- /dev/null +++ b/src/plugins/AIComponents/styling/voice-visualization.scss @@ -0,0 +1,38 @@ +// Intentionally declares no `--str-chat__ai-*` custom properties; see _tokens.scss. +@use './tokens' as *; + +// VoiceWave and VoiceGlow draw on a canvas, which cannot read CSS variables. The script reads +// the computed `color` as the default palette and the computed `text-decoration-color` as the +// backdrop it decides light or dark from. The root holds no text, so neither ever paints. +.str-chat__ai-voice-visualization { + position: relative; + display: block; + min-width: 0; + min-height: 0; + color: ai-token(voice-color); + text-decoration-color: ai-token(voice-backdrop); + -webkit-tap-highlight-color: transparent; + + &__canvas { + position: absolute; + inset: 0; + display: block; + width: 100%; + height: 100%; + } + + &:focus-visible { + @include ai-focus-ring; + } +} + +// No size of their own, so a default that fits each shape when the integrator gives none. +.str-chat__ai-voice-wave { + width: 100%; + height: 8rem; +} + +.str-chat__ai-voice-glow { + width: 12rem; + height: 12rem; +} From 083a9b77724604780a92e66ddff9090ac0e01fe8 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 21 Oct 2026 10:06:09 +0200 Subject: [PATCH 2/8] perf(ai-components): draw the voice glow at 1x and clip the Canvas glow to the globe The glow is all soft gradients, so a 2x backing store filled four times the pixels for no visible difference; it now draws at no more than 1x, while the wave keeps up to 2x for a crisp line. The Canvas glow clips its color masses to the globe instead of painting past it and masking the excess away, and computes the orbit angles once a frame. --- .../VoiceVisualization/VoiceGlow.tsx | 3 ++ .../VoiceVisualizationView.tsx | 13 +++++-- .../renderers/glowCanvasRenderer.ts | 35 ++++++++++++------- .../__tests__/VoiceVisualization.test.tsx | 27 ++++++++++++++ 4 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx index 91186c1d2..0a158e0f4 100644 --- a/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx @@ -52,6 +52,9 @@ export const VoiceGlow = ({ contains={glowContains} createRenderer={createRenderer} data-rendering={webGL ? 'webgl' : 'canvas'} + // The glow is all soft gradients: drawn at 1x it looks the same as at 2x, with a quarter + // of the pixels to fill every frame. + maxPixelRatio={1} rendererKey={webGL ? 'webgl' : 'canvas'} {...props} /> diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx index 2d1749946..239449011 100644 --- a/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx @@ -14,7 +14,8 @@ import type { Hsv, VoiceAgentState, VoiceVisualizationProps } from './types'; const FALLBACK_COLOR = '#005fff'; /** How many frames pass between two reads of the theme, about once a second at 60 Hz. */ const THEME_FRAMES = 60; -const MAX_PIXEL_RATIO = 2; +/** Sharper than this is invisible on a soft drawing and only costs fill rate. */ +const DEFAULT_MAX_PIXEL_RATIO = 2; type ViewProps = VoiceVisualizationProps & { /** Whether a point, centred and normalized to the shorter side with y up, is on the shape. */ @@ -23,6 +24,11 @@ type ViewProps = VoiceVisualizationProps & { createRenderer: (canvas: HTMLCanvasElement) => VoiceRenderer | null; /** Remounts the canvas and remakes the renderer when it changes. */ rendererKey: string; + /** + * The most device pixels per CSS pixel the canvas gets. A blurred shape looks the same at 1, + * at a quarter of the pixels of a 2x display. + */ + maxPixelRatio?: number; }; const useStateLabel = (state: VoiceAgentState) => { @@ -51,6 +57,7 @@ export const VoiceVisualizationView = ({ contains, createRenderer, expressivity, + maxPixelRatio = DEFAULT_MAX_PIXEL_RATIO, onPointerLeave, onPointerMove, onPointerUp, @@ -127,7 +134,7 @@ export const VoiceVisualizationView = ({ let pixelRatio = 1; const resize = () => { - pixelRatio = Math.min(window.devicePixelRatio || 1, MAX_PIXEL_RATIO); + pixelRatio = Math.min(window.devicePixelRatio || 1, maxPixelRatio); const width = Math.round(root.clientWidth * pixelRatio); const height = Math.round(root.clientHeight * pixelRatio); if (canvas.width !== width) canvas.width = width; @@ -210,7 +217,7 @@ export const VoiceVisualizationView = ({ colorScheme?.removeEventListener?.('change', onSchemeChange); renderer.dispose?.(); }; - }, [applyInput, makeRenderer, readTheme, rendererKey]); + }, [applyInput, makeRenderer, maxPixelRatio, readTheme, rendererKey]); /** The pointer in the renderers' coordinates: centred, normalized to the shorter side, y up. */ const toLocal = (event: PointerEvent) => { diff --git a/src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts b/src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts index d4f95bd30..b6e635ba5 100644 --- a/src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts +++ b/src/plugins/AIComponents/VoiceVisualization/renderers/glowCanvasRenderer.ts @@ -28,6 +28,7 @@ export const createGlowCanvasRenderer = ( const rim = [0, 0, 0]; const tmp = [0, 0, 0]; const tmp2 = [0, 0, 0]; + const angles = [0, 0, 0, 0]; /** Slot `index` fading in over a lighter or darker first color as the palette grows. */ const mixSlot = ( @@ -121,15 +122,16 @@ export const createGlowCanvasRenderer = ( const gain = d.bright * mix(1, 0.45 + 0.35 * Math.sin(t * 2), d.load); colors(d, dark); + // Each orbit's angle, once a frame rather than once a blob. + const flowTurn = d.flowSpin * 0.3; + angles[0] = d.orbSpin * 0.42 + 2 * noise(t * 0.12, 0) + flowTurn; + angles[1] = -d.orbSpin * 0.34 + 2 * noise(t * 0.1, 5) + 2 + flowTurn; + angles[2] = d.orbSpin * 0.3 + 2 * noise(t * 0.11, 9) + 1 + flowTurn; + angles[3] = -d.orbSpin * 0.26 + 2 * noise(t * 0.09, 14) + 3.5 + flowTurn; + const blob = (i: number, color: number[], alpha: number) => { if (alpha <= 0.002) return; - const angleByIndex = [ - d.orbSpin * 0.42 + 2 * noise(t * 0.12, 0), - -d.orbSpin * 0.34 + 2 * noise(t * 0.1, 5) + 2, - d.orbSpin * 0.3 + 2 * noise(t * 0.11, 9) + 1, - -d.orbSpin * 0.26 + 2 * noise(t * 0.09, 14) + 3.5, - ]; - const angle = angleByIndex[i] + d.flowSpin * 0.3; + const angle = angles[i]; const distance = 0.4 - 0.02 * i; // A slow breath stands in for the shader's per-pixel warp. const blobRadius = REACH[i] * r * (1 + 0.12 * energy * noise(t * 0.5 + i, 33)); @@ -148,8 +150,14 @@ export const createGlowCanvasRenderer = ( ); }; + const edge = r * 1.06; withScale(d.scale, cx, cy, () => { - circle(cx, cy, r * 1.06, rgba(base, 0.15 * gain)); + // Nothing past the soft edge survives its mask, so the blobs are not painted there at all: + // they reach well beyond the globe, and painting less is most of the cost. + ctx.beginPath(); + ctx.arc(cx, cy, edge, 0, Math.PI * 2); + ctx.clip(); + circle(cx, cy, edge, rgba(base, 0.15 * gain)); const blobAlpha = 0.6 * (0.75 + 0.25 * energy) * gain; blob(1, colB, blobAlpha * 0.8); blob(0, k0, blobAlpha); @@ -178,16 +186,17 @@ export const createGlowCanvasRenderer = ( ); }); - // The soft edge, outside the bounce's scale so it covers the whole area at any scale and - // whatever spilled past the globe goes too. The canvas is the glow's own layer, so this masks - // only the glow, never what is behind it. + // The soft edge, outside the bounce's scale so it follows the globe at any scale. The canvas + // is the glow's own layer, so this masks only the glow, never what is behind it. The clip + // above left nothing outside the globe, so only its square needs masking. + const masked = edge * d.scale; ctx.globalCompositeOperation = 'destination-in'; - ctx.fillStyle = radial(cx, cy, r * 1.06 * d.scale, [ + ctx.fillStyle = radial(cx, cy, masked, [ [0, 'rgba(0, 0, 0, 1)'], [0.62 / 1.06, 'rgba(0, 0, 0, 1)'], [1, 'rgba(0, 0, 0, 0)'], ]); - ctx.fillRect(0, 0, width, height); + ctx.fillRect(cx - masked, cy - masked, masked * 2, masked * 2); // The halo, behind the globe. ctx.globalCompositeOperation = 'destination-over'; diff --git a/src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx b/src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx index 261532a93..be4fbaffa 100644 --- a/src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx +++ b/src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx @@ -248,6 +248,33 @@ describe('VoiceWave and VoiceGlow', () => { expect(frames).toHaveLength(0); }); + it('draws the soft glow at 1x and the sharp wave at up to 2x', () => { + vi.spyOn(window, 'devicePixelRatio', 'get').mockReturnValue(3); + render( + <> + + + , + ); + const [glow, wave] = screen + .getAllByRole('img') + .map((root) => root.querySelector('canvas') as HTMLCanvasElement); + + expect([glow.width, glow.height]).toEqual([200, 200]); + expect([wave.width, wave.height]).toEqual([400, 400]); + }); + + it('paints the Canvas glow only inside the globe', () => { + render(); + step(); + + expect(calls.some((call) => call.name === 'clip')).toBe(true); + const mask = calls.find( + (call) => call.name === 'fillRect' && (call.args[2] as number) < 200, + ); + expect(mask).toBeDefined(); + }); + it('stops the loop on unmount', () => { const { unmount } = render(); step(); From d715e397f1437a02bc7cf292a85397e0856524a1 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 21 Oct 2026 13:31:33 +0200 Subject: [PATCH 3/8] feat(ai-components): add a CSS renderer for VoiceGlow `rendering='css'` composes the glow the way the shader does, as CSS: an opaque color field (a base and the masses mixed over it in order, with the shader's smoothstep falloffs, diffuse shade and fresnel rim) seen through an alpha that only the first two masses carry, a faint static body plus one mask riding each of their orbits, with inverse turns inside each mask keeping the colors still. Every gradient is painted once and only transform and opacity animate, so the compositor runs the motion: it keeps moving while the main thread is busy, needs no canvas or WebGL, and runs no script per frame unless an audio level is given. The palette is computed with the shader's color math and follows the theme; each state sets playback rates, so state changes ease instead of jumping. WebGL stays the default for `auto`. --- src/plugins/AIComponents/README.md | 17 +- .../VoiceVisualization/VoiceGlow.tsx | 8 +- .../VoiceVisualization/VoiceGlowCss.tsx | 329 +++++++++++ .../VoiceVisualizationView.tsx | 18 +- .../VoiceVisualization/cssGlow.ts | 187 ++++++ .../AIComponents/VoiceVisualization/types.ts | 5 +- .../VoiceVisualization/useVoiceStateLabel.ts | 15 + .../__tests__/VoiceGlowCss.test.tsx | 295 ++++++++++ .../AIComponents/__tests__/a11y.test.tsx | 4 + .../__tests__/classNames.test.tsx | 2 + .../AIComponents/__tests__/hydration.test.tsx | 4 +- .../AIComponents/__tests__/ssr.test.tsx | 4 +- .../__tests__/styleTokens.test.ts | 13 + src/plugins/AIComponents/styling/index.scss | 1 + .../AIComponents/styling/voice-glow-css.scss | 552 ++++++++++++++++++ 15 files changed, 1428 insertions(+), 26 deletions(-) create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/cssGlow.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/useVoiceStateLabel.ts create mode 100644 src/plugins/AIComponents/__tests__/VoiceGlowCss.test.tsx create mode 100644 src/plugins/AIComponents/styling/voice-glow-css.scss diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md index c283e426d..126b8cc5f 100644 --- a/src/plugins/AIComponents/README.md +++ b/src/plugins/AIComponents/README.md @@ -308,13 +308,13 @@ Both draw on a `` inside a `div` with `role="img"` and an accessible nam #### Props -| Name | Type | Required | Description | -| -------------- | ------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `state` | `'idle' \| 'connecting' \| 'listening' \| 'thinking' \| 'speaking'` | yes | What the agent is doing. The states differ only in motion, never in color, and a change eases from one motion into the next. | -| `colors` | `string[]` | no | One to five CSS colors, flowing through the shape. Past five are ignored. When omitted or empty, `--str-chat__ai-voice-color` is used, which follows the SDK accent color. | -| `expressivity` | `number` | no | From 0 to 2: how far and how fast it moves. Defaults to 1. | -| `audioLevel` | `() => number \| null \| undefined` | no | The level of whoever is talking, from 0 to 1, read once a frame, so a level changing many times a second re-renders nothing. Returning `null` (or omitting it) animates speech on its own. | -| `rendering` | `'auto' \| 'webgl' \| 'canvas'` | no | `VoiceGlow` only. `auto` (default) and `webgl` draw with a WebGL shader and fall back to a 2D canvas when WebGL is unavailable or its context is lost; `canvas` always uses the cheaper 2D drawing. | +| Name | Type | Required | Description | +| -------------- | ------------------------------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `state` | `'idle' \| 'connecting' \| 'listening' \| 'thinking' \| 'speaking'` | yes | What the agent is doing. The states differ only in motion, never in color, and a change eases from one motion into the next. | +| `colors` | `string[]` | no | One to five CSS colors, flowing through the shape. Past five are ignored. When omitted or empty, `--str-chat__ai-voice-color` is used, which follows the SDK accent color. | +| `expressivity` | `number` | no | From 0 to 2: how far and how fast it moves. Defaults to 1. | +| `audioLevel` | `() => number \| null \| undefined` | no | The level of whoever is talking, from 0 to 1, read once a frame, so a level changing many times a second re-renders nothing. Returning `null` (or omitting it) animates speech on its own. | +| `rendering` | `'auto' \| 'webgl' \| 'canvas' \| 'css'` | no | `VoiceGlow` only. `auto` (default) and `webgl` draw with a WebGL shader and fall back to a 2D canvas when WebGL is unavailable or its context is lost; `canvas` always uses the cheaper 2D drawing; `css` builds the glow from CSS gradients animated by the browser's compositor (see below). | Any other `div` prop (`className`, `style`, event handlers, ...) is passed to the root. Pointer events are watched, never stopped: a tap inside the shape bounces it and a hovering mouse or pen ripples it, while your own `onClick` still fires. @@ -323,6 +323,9 @@ Behavior: - The animation runs only while the element is on screen and the tab is visible. - With `prefers-reduced-motion: reduce`, the motion slows to a crawl and taps and hover are ignored. - The palette and the light or dark look follow the theme (`--str-chat__ai-voice-color` and `--str-chat__ai-voice-backdrop`), re-read about once a second, so a theme switch needs no prop. + +`rendering='css'` composes the glow the way the shader does, as gradients that are painted once and animated only through `transform` and `opacity`, so the browser's compositor runs the motion. It keeps moving while the main thread is busy (for example while a long answer streams in), needs no canvas or WebGL, and runs no script per frame unless `audioLevel` is given. It lacks the shader's fine marbling: a tap bounces the glow and sends out a ring instead of a ripple, and a hovering pointer does nothing. It needs `@property`, container query units and `color-mix()` (Chrome 111, Safari 16.4, Firefox 128 and later). + - Under SSR the markup is rendered and drawing starts after hydration. #### Example diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx index 0a158e0f4..25fdf81f9 100644 --- a/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx @@ -4,6 +4,7 @@ import clsx from 'clsx'; import { createGlowCanvasRenderer, glowContains } from './renderers/glowCanvasRenderer'; import { createGlowWebGLRenderer } from './renderers/glowWebGLRenderer'; import type { VoiceGlowRendering, VoiceVisualizationProps } from './types'; +import { VoiceGlowCss } from './VoiceGlowCss'; import { VoiceVisualizationView } from './VoiceVisualizationView'; export type VoiceGlowProps = VoiceVisualizationProps & { @@ -12,7 +13,8 @@ export type VoiceGlowProps = VoiceVisualizationProps & { }; /** Whether `rendering` tries WebGL first; any WebGL failure still falls back to Canvas. */ -export const usesWebGL = (rendering: VoiceGlowRendering) => rendering !== 'canvas'; +export const usesWebGL = (rendering: VoiceGlowRendering) => + rendering === 'auto' || rendering === 'webgl'; const createCanvasRenderer = (canvas: HTMLCanvasElement) => { const ctx = canvas.getContext('2d'); @@ -46,6 +48,10 @@ export const VoiceGlow = ({ [webGL], ); + if (rendering === 'css') { + return ; + } + return ( ( +
+ {children} +
+); + +const orbitStyle = (orbit: number) => + ({ + '--str-ai-glow-direction': ORBITS[orbit].direction, + '--str-ai-glow-period': `${ORBITS[orbit].period}s`, + '--str-ai-glow-start': `${ORBITS[orbit].start}deg`, + }) as CSSProperties; + +const swayStyle = (orbit: number) => + ({ '--str-ai-glow-sway-period': `${ORBITS[orbit].sway}s` }) as CSSProperties; + +const warpStyle = (orbit: number, [period, turn]: [number, number]) => + ({ + '--str-ai-glow-dist': orbitDistance(orbit), + '--str-ai-glow-warp-period': `${period}s`, + '--str-ai-glow-warp-turn': `${turn}deg`, + }) as CSSProperties; + +/** The opaque color field: a base, the masses mixed over it in order, then shade and rim. */ +const ColorField = () => ( + + + + {MASSES.map((mass, index) => ( + + + +
+ + + + ))} + + + + +); + +/** + * The color field seen through opacity that rides an orbit: the mask turns and stretches with + * the orbit, and the inverse turns and stretches inside it keep the colors where they are. + */ +const RidingBody = ({ alpha, orbit, reach, warp }: (typeof RIDING_BODIES)[number]) => ( + + + + +
+ + + + + + + + + +
+ + + + +); + +type Ripple = { id: number; x: number; y: number }; + +/** + * VoiceGlow drawn with CSS: the same composition as the shader, with every gradient painted + * once and only `transform` and `opacity` animated, so the browser's compositor runs the + * motion. It keeps moving while the page is busy, needs no canvas or WebGL, and costs no + * JavaScript per frame unless `audioLevel` is given. + * + * Script sets what the stylesheet cannot know: the palette (read from the theme when `colors` + * is not given), the playback rate of each animation for the state, and the live audio level. + */ +export const VoiceGlowCss = ({ + audioLevel, + className, + colors, + expressivity, + onPointerUp, + state, + style, + ...restProps +}: VoiceVisualizationProps) => { + const label = useVoiceStateLabel(state); + const reduceMotion = useReducedMotionPreference(); + const rootRef = useRef(null); + const bounceRef = useRef(null); + const audioLevelRef = useRef(audioLevel); + const [ripples, setRipples] = useState([]); + const { amplitude, speed } = scales(expressivityInput(expressivity)); + const colorsKey = (colors ?? []).join('|'); + const hasAudioLevel = !!audioLevel; + + useEffect(() => { + audioLevelRef.current = audioLevel; + }); + + // The palette, from `colors` or the theme, and whether the glow sits on a dark backdrop. Read + // again about once a second, so a theme switch is followed without a prop. + useEffect(() => { + const root = rootRef.current; + if (!root || typeof getComputedStyle !== 'function') return; + const own = colorsKey ? colorsKey.split('|') : []; + let applied = ''; + const apply = () => { + const computed = getComputedStyle(root); + const backdrop = luminance(computed.textDecorationColor || ''); + const dark = backdrop !== null && backdrop < 0.5 ? 1 : 0; + let palette = toHsvPalette(own.length ? own : [computed.color]); + if (!palette.length) palette = toHsvPalette([FALLBACK_COLOR]); + const properties = cssGlowPalette(palette, dark); + const next = JSON.stringify(properties) + dark; + if (next === applied) return; + applied = next; + for (const [name, value] of Object.entries(properties)) { + root.style.setProperty(name, value); + } + root.setAttribute('data-dark', String(!!dark)); + }; + apply(); + const interval = setInterval(apply, THEME_INTERVAL); + const scheme = + typeof window.matchMedia === 'function' + ? window.matchMedia('(prefers-color-scheme: dark)') + : null; + scheme?.addEventListener?.('change', apply); + return () => { + clearInterval(interval); + scheme?.removeEventListener?.('change', apply); + }; + }, [colorsKey]); + + // Each state plays the animations at its own rates. Changing a playback rate keeps an + // animation where it is, where changing its duration would make it jump; a turn and its + // inverse always get the same rate, so they stay in step. + useEffect(() => { + const root = rootRef.current; + if (!root || typeof root.getAnimations !== 'function') return; + const rates = CSS_GLOW_RATES[state]; + const slow = reduceMotion ? REDUCED_MOTION_RATE : 1; + for (const animation of root.getAnimations({ subtree: true })) { + const kind = cssGlowRateKind((animation as CSSAnimation).animationName ?? ''); + if (kind) animation.updatePlaybackRate(rates[kind] * speed * slow); + } + }, [reduceMotion, speed, state]); + + // A live audio level, eased like the driver eases it, is the only value written per frame. + useEffect(() => { + const root = rootRef.current; + if (!hasAudioLevel || !root || typeof requestAnimationFrame !== 'function') return; + let frame = 0; + let last = 0; + let level = 0; + let written = -1; + let live: boolean | null = null; + const tick = (now: number) => { + const target = audioLevelInput(audioLevelRef.current?.()); + const dt = last ? Math.min((now - last) / 1000, 0.05) : 0; + last = now; + level += ((target ?? 0) - level) * ease(MIC_EASE, dt); + if (Math.abs(level - written) > 0.002) { + written = level; + root.style.setProperty('--str-ai-glow-level', level.toFixed(3)); + } + if (live !== (target !== null)) { + live = target !== null; + root.setAttribute('data-live', String(live)); + } + frame = requestAnimationFrame(tick); + }; + frame = requestAnimationFrame(tick); + return () => { + cancelAnimationFrame(frame); + root.style.removeProperty('--str-ai-glow-level'); + root.removeAttribute('data-live'); + }; + }, [hasAudioLevel]); + + const rippleTimers = useRef[]>([]); + useEffect(() => () => rippleTimers.current.forEach(clearTimeout), []); + + // A tap inside the globe bounces it and sends a ring out from the touch. Nothing is + // prevented or stopped, so a click handler on the glow or a parent still gets it. + const handlePointerUp = (event: PointerEvent) => { + onPointerUp?.(event); + if (reduceMotion) return; + const rect = event.currentTarget.getBoundingClientRect(); + const side = Math.min(rect.width, rect.height) || 1; + const x = event.clientX - rect.left; + const y = event.clientY - rect.top; + if (!glowContains((x - rect.width / 2) / side, (rect.height / 2 - y) / side)) return; + const bounce = bounceRef.current; + if (bounce) { + bounce.classList.remove(`${block}__bounce--active`); + // Restarts the bounce when taps come faster than it lasts. + void bounce.offsetWidth; + bounce.classList.add(`${block}__bounce--active`); + } + const id = Date.now() + Math.random(); + setRipples((current) => [...current, { id, x, y }]); + rippleTimers.current.push( + setTimeout( + () => setRipples((current) => current.filter((ripple) => ripple.id !== id)), + RIPPLE_DURATION, + ), + ); + }; + + return ( +
+ + +
+ + + + + +
+ +
+ {RIDING_BODIES.map((body) => ( + + ))} + + + + + +
+ + {ripples.map((ripple) => ( +
+ ); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx index 239449011..b795a0653 100644 --- a/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx @@ -1,14 +1,14 @@ import { type PointerEvent, useEffect, useRef } from 'react'; import clsx from 'clsx'; -import { useTranslationContext } from '../../../context/TranslationContext'; import { useReducedMotionPreference } from '../../../components/MessageList/hooks/useReducedMotionPreference'; import { useStableCallback } from '../../../utils/useStableCallback'; import { type VoiceDriverInput, VoiceVisualizationDriver } from './driver'; import { audioLevelInput, expressivityInput } from './math'; import { luminance, toHsvPalette } from './palette'; import type { VoiceRenderer } from './renderers/types'; -import type { Hsv, VoiceAgentState, VoiceVisualizationProps } from './types'; +import type { Hsv, VoiceVisualizationProps } from './types'; +import { useVoiceStateLabel } from './useVoiceStateLabel'; /** Used when neither the `colors` prop nor the theme gives a readable color. */ const FALLBACK_COLOR = '#005fff'; @@ -31,18 +31,6 @@ type ViewProps = VoiceVisualizationProps & { maxPixelRatio?: number; }; -const useStateLabel = (state: VoiceAgentState) => { - const { t } = useTranslationContext(); - const labels: Record = { - connecting: t('Connecting'), - idle: t('Ready'), - listening: t('Listening'), - speaking: t('Speaking'), - thinking: t('Thinking'), - }; - return t('aria/Voice agent: {{ state }}', { state: labels[state] }); -}; - /** * What VoiceWave and VoiceGlow share: the motion advanced once a frame, the canvas kept at the * element's size, the theme's colors, reduced motion, the pointer and the accessible name. @@ -65,7 +53,7 @@ export const VoiceVisualizationView = ({ state, ...restProps }: ViewProps) => { - const label = useStateLabel(state); + const label = useVoiceStateLabel(state); const reduceMotion = useReducedMotionPreference(); const rootRef = useRef(null); const canvasRef = useRef(null); diff --git a/src/plugins/AIComponents/VoiceVisualization/cssGlow.ts b/src/plugins/AIComponents/VoiceVisualization/cssGlow.ts new file mode 100644 index 000000000..2a04e4b36 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/cssGlow.ts @@ -0,0 +1,187 @@ +import { clamp, mix, saturate, vivid } from './math'; +import type { Hsv, VoiceAgentState } from './types'; + +/* + * The CSS glow is built the way the shader composes the glow: an opaque color field (a base and + * the masses mixed over it in order, with smoothstep falloffs, then shaded) seen through an alpha + * that only the first two masses make, 0.15 + 0.6 (b0 + 0.8 b1). The gradients below are what + * the stylesheet cannot know ahead of time: the falloffs, the masks and the palette. + */ + +/** R as a share of the globe's width: the globe is the soft edge, 1.06 R across each side. */ +const R = 100 / 2.12; + +/** + * The four orbits, from the shader's angles: the period of a turn at playback rate 1 (2π over its + * share of the orbit spin), its direction, starting angle and distance from the centre in R, and + * how long a sway takes in place of the noise that wobbles it. + */ +export const ORBITS = [ + { direction: 'normal', distance: 0.4, period: 15, start: 0, sway: 8.3 }, + { direction: 'reverse', distance: 0.38, period: 18.5, start: 115, sway: 10.1 }, + { direction: 'normal', distance: 0.36, period: 21, start: 57, sway: 9.1 }, + { direction: 'reverse', distance: 0.34, period: 24, start: 200, sway: 11.3 }, +] as const; + +type Mass = { + color: string; + orbit: number; + reach: number; + /** Period and turn of the stretch that stands in for the noise bending it. */ + warp: [number, number]; + weight?: string; +}; + +/** The masses in the order the shader mixes them, each reaching `reach` R. */ +export const MASSES: Mass[] = [ + { color: 'var(--str-ai-glow-k0)', orbit: 0, reach: 1.15, warp: [5.3, 20] }, + { color: 'var(--str-ai-glow-col-b)', orbit: 1, reach: 1.3, warp: [6.7, 75] }, + { + color: 'var(--str-ai-glow-col-c)', + orbit: 0, + reach: 1.15, + warp: [4.9, -30], + weight: '0.5', + }, + { + color: 'var(--str-ai-glow-col-d)', + orbit: 2, + reach: 1.2, + warp: [7.9, 110], + weight: 'var(--str-ai-glow-w3, 0)', + }, + { + color: 'var(--str-ai-glow-col-e)', + orbit: 3, + reach: 1.25, + warp: [6.1, -60], + weight: 'var(--str-ai-glow-w4, 0)', + }, +]; + +/** The two masses that carry the glow's opacity: orbit, reach, peak alpha and their stretch. */ +export const RIDING_BODIES = [ + { alpha: 0.6, orbit: 0, reach: 1.15, warp: [6.3, 35] as [number, number] }, + { alpha: 0.48, orbit: 1, reach: 1.3, warp: [7.7, -50] as [number, number] }, +]; + +const smoothstep = (edge0: number, edge1: number, x: number) => { + const t = clamp((x - edge0) / (edge1 - edge0), 0, 1); + return t * t * (3 - 2 * t); +}; + +/** smoothstep(reach, 0.1, x) over 0..reach as gradient stops, each `paint(weight, offset)`. */ +const falloff = (reach: number, paint: (weight: number, offset: string) => string) => { + const stops: string[] = []; + for (let k = 0; k <= 10; k++) { + const x = (reach * k) / 10; + stops.push(paint(smoothstep(reach, 0.1, x), `${((x / reach) * 100).toFixed(1)}%`)); + } + return stops.join(', '); +}; + +/** A mass: its color weighted by the falloff, on its own circle. */ +export const massBackground = ({ color, reach, weight = '1' }: Mass) => + `radial-gradient(closest-side, ${falloff( + reach, + (w, at) => + `color-mix(in srgb, ${color} calc(${weight} * ${(w * 100).toFixed(1)}%), transparent) ${at}`, + )})`; + +/** The opacity a mass carries along its orbit: its peak alpha times its falloff. */ +export const orbitMask = (orbit: number, reach: number, alpha: number) => { + const size = `${(reach * R).toFixed(2)}%`; + const centre = `${(50 + ORBITS[orbit].distance * R).toFixed(2)}% 50%`; + return `radial-gradient(ellipse ${size} ${size} at ${centre}, ${falloff( + reach, + (w, at) => `rgb(0 0 0 / ${(alpha * w).toFixed(3)}) ${at}`, + )})`; +}; + +/** How far a mass sits from the centre, as a share of the globe. */ +export const orbitDistance = (orbit: number) => + `${(ORBITS[orbit].distance * R).toFixed(2)}%`; + +type Rgb = [number, number, number]; + +const rgb = ([r, g, b]: Rgb) => + `rgb(${Math.round(r * 255)} ${Math.round(g * 255)} ${Math.round(b * 255)})`; + +const scaled = (c: Rgb, k: number): Rgb => [ + clamp(c[0] * k, 0, 1), + clamp(c[1] * k, 0, 1), + clamp(c[2] * k, 0, 1), +]; + +const saturated = (c: Rgb, amount: number): Rgb => { + const out = [...c]; + saturate(out, amount); + return out as Rgb; +}; + +const towardWhite = (c: Rgb, amount: number): Rgb => [ + mix(c[0], 1, amount), + mix(c[1], 1, amount), + mix(c[2], 1, amount), +]; + +/** The field's last steps, saturate3(col, 1.15) and x1.12, folded into each color: both linear. */ +const finish = (c: Rgb) => scaled(saturated(c, 1.15), 1.12); + +/** + * The colors the CSS glow mixes, as the shader derives them from the palette: masses saturated + * x1.7, the first color lightened or darkened where the palette has no color of its own, a + * near-white tint of it for the base, the halo and the rim. In custom properties for the + * stylesheet. + */ +export const cssGlowPalette = (palette: Hsv[], dark: number): Record => { + if (!palette.length) return {}; + const colors = palette.map((color) => { + const out = [0, 0, 0]; + vivid(color, dark, out); + return out as Rgb; + }); + const [first] = colors; + const mass = (c: Rgb) => rgb(finish(saturated(c, 1.7))); + return { + '--str-ai-glow-base': rgb(finish(saturated(towardWhite(first, 0.06), 1.3))), + '--str-ai-glow-col-b': colors[1] + ? mass(colors[1]) + : rgb(finish(saturated(scaled(first, 1.18), 1.3))), + '--str-ai-glow-col-c': colors[2] + ? mass(colors[2]) + : rgb(finish(saturated(scaled(first, 0.85), 1.3))), + '--str-ai-glow-col-d': mass(colors[3] ?? first), + '--str-ai-glow-col-e': mass(colors[4] ?? first), + '--str-ai-glow-halo': rgb(towardWhite(first, dark ? 0.32 : 0.75)), + '--str-ai-glow-k0': mass(first), + '--str-ai-glow-rim': rgb(towardWhite(saturated(first, 1.7), 0.4)), + '--str-ai-glow-shade': 'rgb(0 0 0)', + '--str-ai-glow-w3': colors.length > 3 ? '1' : '0', + '--str-ai-glow-w4': colors.length > 4 ? '1' : '0', + }; +}; + +/** + * Playback rates per state, from the driver's motion: `orbit` how fast the masses turn, `flow` + * the slow turn of thinking, `tempo` everything else. + */ +export const CSS_GLOW_RATES: Record< + VoiceAgentState, + { flow: number; orbit: number; tempo: number } +> = { + connecting: { flow: 0, orbit: 1, tempo: 0.8 }, + idle: { flow: 0, orbit: 0.405, tempo: 0.32 }, + listening: { flow: 0, orbit: 0.94, tempo: 0.72 }, + speaking: { flow: 0, orbit: 1.375, tempo: 1 }, + thinking: { flow: 1, orbit: 1.875, tempo: 1 }, +}; + +/** Which rate an animation of the stylesheet plays at, or null for one-off animations. */ +export const cssGlowRateKind = (animationName: string) => { + if (animationName.endsWith('-flow') || animationName.endsWith('-unflow')) return 'flow'; + if (animationName.endsWith('-spin') || animationName.endsWith('-unspin')) + return 'orbit'; + if (animationName.endsWith('-bounce') || animationName.endsWith('-ripple')) return null; + return 'tempo'; +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/types.ts b/src/plugins/AIComponents/VoiceVisualization/types.ts index eb64ea4d3..a5db838d5 100644 --- a/src/plugins/AIComponents/VoiceVisualization/types.ts +++ b/src/plugins/AIComponents/VoiceVisualization/types.ts @@ -27,8 +27,11 @@ export type VoiceAgentState = * WebGL is unavailable or its context is lost. * - `canvas`: a handful of 2D gradients, cheaper, without the shader's marbling or its ripples * from a tap or a hovering pointer. + * - `css`: the shader's composition in CSS, animated by the browser's compositor. It keeps + * moving while the page is busy and needs no canvas, at the cost of the shader's fine + * marbling; a tap sends out a ring instead of a ripple, and a hovering pointer does nothing. */ -export type VoiceGlowRendering = 'auto' | 'webgl' | 'canvas'; +export type VoiceGlowRendering = 'auto' | 'webgl' | 'canvas' | 'css'; export type VoiceVisualizationProps = Omit< ComponentPropsWithoutRef<'div'>, diff --git a/src/plugins/AIComponents/VoiceVisualization/useVoiceStateLabel.ts b/src/plugins/AIComponents/VoiceVisualization/useVoiceStateLabel.ts new file mode 100644 index 000000000..7678881a0 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/useVoiceStateLabel.ts @@ -0,0 +1,15 @@ +import { useTranslationContext } from '../../../context/TranslationContext'; +import type { VoiceAgentState } from './types'; + +/** The accessible name of a voice visualization: "Voice agent" and what it is doing. */ +export const useVoiceStateLabel = (state: VoiceAgentState) => { + const { t } = useTranslationContext(); + const labels: Record = { + connecting: t('Connecting'), + idle: t('Ready'), + listening: t('Listening'), + speaking: t('Speaking'), + thinking: t('Thinking'), + }; + return t('aria/Voice agent: {{ state }}', { state: labels[state] }); +}; diff --git a/src/plugins/AIComponents/__tests__/VoiceGlowCss.test.tsx b/src/plugins/AIComponents/__tests__/VoiceGlowCss.test.tsx new file mode 100644 index 000000000..cdcd262d3 --- /dev/null +++ b/src/plugins/AIComponents/__tests__/VoiceGlowCss.test.tsx @@ -0,0 +1,295 @@ +import React from 'react'; +import { act, fireEvent, render, screen } from '@testing-library/react'; + +import { VoiceGlow } from '..'; +import { + CSS_GLOW_RATES, + cssGlowPalette, + cssGlowRateKind, +} from '../VoiceVisualization/cssGlow'; +import { toHsvPalette } from '../VoiceVisualization/palette'; + +type FakeAnimation = { + animationName: string; + updatePlaybackRate: ReturnType; +}; + +const animation = (animationName: string): FakeAnimation => ({ + animationName, + updatePlaybackRate: vi.fn(), +}); + +describe('VoiceGlow rendering="css"', () => { + let frames: FrameRequestCallback[]; + let now: number; + let animations: FakeAnimation[]; + + const step = (count = 1) => { + for (let i = 0; i < count; i++) { + const pending = frames; + frames = []; + now += 1000 / 60; + act(() => pending.forEach((callback) => callback(now))); + } + }; + + const lastRate = (name: string) => + animations + .find((a) => a.animationName === name) + ?.updatePlaybackRate.mock.calls.at(-1)?.[0]; + + beforeEach(() => { + frames = []; + now = 1000; + animations = [ + 'str-ai-glow-spin', + 'str-ai-glow-unspin', + 'str-ai-glow-flow', + 'str-ai-glow-unflow', + 'str-ai-glow-breath', + 'str-ai-glow-bounce', + ].map(animation); + vi.spyOn(window, 'requestAnimationFrame').mockImplementation((callback) => { + frames.push(callback); + return frames.length; + }); + vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => undefined); + // jsdom has no Web Animations; stand in for the stylesheet's running animations. + Object.defineProperty(HTMLElement.prototype, 'getAnimations', { + configurable: true, + value: () => animations, + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + delete (HTMLElement.prototype as { getAnimations?: unknown }).getAnimations; + }); + + it('draws with elements, not a canvas, under the same name and root classes', () => { + const getContext = vi.spyOn(HTMLCanvasElement.prototype, 'getContext'); + const { container } = render( + , + ); + const glow = screen.getByTestId('glow'); + + expect(glow).toHaveClass( + 'str-chat__ai-voice-visualization', + 'str-chat__ai-voice-glow', + 'str-chat__ai-voice-glow--css', + 'custom', + ); + expect(glow).toHaveAttribute('role', 'img'); + expect(glow).toHaveAttribute('data-rendering', 'css'); + expect(glow).toHaveAttribute('data-state', 'speaking'); + expect(container.querySelector('canvas')).toBeNull(); + expect(getContext).not.toHaveBeenCalledWith('webgl', expect.anything()); + // A faint still body and one riding each of the two orbits, each over the color field. + expect(container.querySelectorAll('.str-chat__ai-voice-glow__field')).toHaveLength(3); + expect(container.querySelectorAll('.str-chat__ai-voice-glow__mass')).toHaveLength(15); + }); + + it('sets the palette from the colors prop', () => { + // The stylesheet sets this to the backdrop token; jsdom loads no stylesheet. + render( + , + ); + const glow = screen.getByRole('img'); + const expected = cssGlowPalette(toHsvPalette(['#005fff', '#7c3aed']), 0); + + expect(glow.style.getPropertyValue('--str-ai-glow-k0')).toBe( + expected['--str-ai-glow-k0'], + ); + expect(glow.style.getPropertyValue('--str-ai-glow-col-b')).toBe( + expected['--str-ai-glow-col-b'], + ); + expect(glow.style.getPropertyValue('--str-ai-glow-w3')).toBe('0'); + expect(glow).toHaveAttribute('data-dark', 'false'); + }); + + it('takes the theme color when no colors are given, and follows a dark backdrop', () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] }); + try { + render( + , + ); + const glow = screen.getByRole('img'); + expect(glow.style.getPropertyValue('--str-ai-glow-k0')).toBe( + cssGlowPalette(toHsvPalette(['rgb(255, 0, 0)']), 0)['--str-ai-glow-k0'], + ); + + glow.style.textDecorationColor = 'rgb(20, 20, 20)'; + act(() => { + vi.advanceTimersByTime(1000); + }); + expect(glow).toHaveAttribute('data-dark', 'true'); + } finally { + vi.useRealTimers(); + } + }); + + it('plays each state at its rates, a turn and its inverse alike', () => { + const { rerender } = render(); + expect(lastRate('str-ai-glow-spin')).toBeCloseTo(CSS_GLOW_RATES.idle.orbit); + expect(lastRate('str-ai-glow-flow')).toBe(0); + + rerender(); + expect(lastRate('str-ai-glow-spin')).toBeCloseTo(CSS_GLOW_RATES.thinking.orbit); + expect(lastRate('str-ai-glow-unspin')).toBe(lastRate('str-ai-glow-spin')); + expect(lastRate('str-ai-glow-flow')).toBeCloseTo(1); + expect(lastRate('str-ai-glow-unflow')).toBe(lastRate('str-ai-glow-flow')); + expect(lastRate('str-ai-glow-breath')).toBeCloseTo(CSS_GLOW_RATES.thinking.tempo); + // A tap's bounce plays as authored. + expect(lastRate('str-ai-glow-bounce')).toBeUndefined(); + }); + + it('scales speed with expressivity and amplitude through a custom property', () => { + render(); + + expect(lastRate('str-ai-glow-spin')).toBeCloseTo(CSS_GLOW_RATES.speaking.orbit * 1.4); + expect(screen.getByRole('img').style.getPropertyValue('--str-ai-glow-amp')).toBe( + '1.850', + ); + }); + + it('writes a live audio level once a frame and marks it live', () => { + const audioLevel = vi.fn(() => 0.8); + render(); + const glow = screen.getByRole('img'); + + step(30); + + expect(audioLevel).toHaveBeenCalledTimes(30); + expect(glow).toHaveAttribute('data-live', 'true'); + expect(Number(glow.style.getPropertyValue('--str-ai-glow-level'))).toBeCloseTo( + 0.8, + 1, + ); + }); + + it('runs no frame loop without an audio level', () => { + render(); + + expect(frames).toHaveLength(0); + }); + + it('bounces on a tap inside the globe, rings, and lets a parent handle the click', () => { + const onClick = vi.fn(); + const onPointerUp = vi.fn(); + const { container } = render( +
+ +
, + ); + const glow = screen.getByRole('img'); + vi.spyOn(glow, 'getBoundingClientRect').mockReturnValue({ + height: 200, + left: 0, + top: 0, + width: 200, + } as DOMRect); + + fireEvent.pointerUp(glow, { clientX: 2, clientY: 2 }); + expect( + container.querySelector('.str-chat__ai-voice-glow__bounce--active'), + ).toBeNull(); + + fireEvent.pointerUp(glow, { clientX: 100, clientY: 110 }); + fireEvent.click(glow); + + expect( + container.querySelector('.str-chat__ai-voice-glow__bounce--active'), + ).toBeInTheDocument(); + expect(container.querySelectorAll('.str-chat__ai-voice-glow__ripple')).toHaveLength( + 1, + ); + expect(onPointerUp).toHaveBeenCalledTimes(2); + expect(onClick).toHaveBeenCalledTimes(1); + }); + + it('does not bounce under reduced motion, and slows every animation', () => { + vi.spyOn(window, 'matchMedia').mockImplementation( + (query: string) => + ({ + addEventListener: () => undefined, + matches: query.includes('reduce'), + media: query, + removeEventListener: () => undefined, + }) as unknown as MediaQueryList, + ); + const { container } = render(); + const glow = screen.getByRole('img'); + vi.spyOn(glow, 'getBoundingClientRect').mockReturnValue({ + height: 200, + left: 0, + top: 0, + width: 200, + } as DOMRect); + + fireEvent.pointerUp(glow, { clientX: 100, clientY: 100 }); + + expect( + container.querySelector('.str-chat__ai-voice-glow__bounce--active'), + ).toBeNull(); + expect(lastRate('str-ai-glow-spin')).toBeCloseTo( + CSS_GLOW_RATES.speaking.orbit * 0.07, + ); + }); +}); + +describe('cssGlowPalette', () => { + const blue = toHsvPalette(['#005fff']); + + it('saturates the masses as the shader does', () => { + // saturate3(c, 1.7), then the field's saturate3(1.15) and x1.12, all clamped. + expect(cssGlowPalette(blue, 0)['--str-ai-glow-k0']).toBe('rgb(0 117 255)'); + }); + + it('fills slots the palette leaves empty from the first color', () => { + const palette = cssGlowPalette(blue, 0); + const twoColors = cssGlowPalette(toHsvPalette(['#005fff', '#7c3aed']), 0); + + expect(palette['--str-ai-glow-col-b']).not.toBe(twoColors['--str-ai-glow-col-b']); + expect(palette['--str-ai-glow-col-d']).toBe(palette['--str-ai-glow-k0']); + expect(palette['--str-ai-glow-w3']).toBe('0'); + expect( + cssGlowPalette(toHsvPalette(['#f00', '#0f0', '#00f', '#ff0', '#0ff']), 0)[ + '--str-ai-glow-w4' + ], + ).toBe('1'); + }); + + it('tints the halo less on a dark backdrop', () => { + expect(cssGlowPalette(blue, 1)['--str-ai-glow-halo']).not.toBe( + cssGlowPalette(blue, 0)['--str-ai-glow-halo'], + ); + }); + + it('returns nothing for an empty palette', () => { + expect(cssGlowPalette([], 0)).toEqual({}); + }); + + it('names which rate each animation plays at', () => { + expect(cssGlowRateKind('str-ai-glow-spin')).toBe('orbit'); + expect(cssGlowRateKind('str-ai-glow-unspin')).toBe('orbit'); + expect(cssGlowRateKind('str-ai-glow-flow')).toBe('flow'); + expect(cssGlowRateKind('str-ai-glow-unflow')).toBe('flow'); + expect(cssGlowRateKind('str-ai-glow-mask-warp')).toBe('tempo'); + expect(cssGlowRateKind('str-ai-glow-bounce')).toBeNull(); + expect(cssGlowRateKind('str-ai-glow-ripple')).toBeNull(); + }); +}); diff --git a/src/plugins/AIComponents/__tests__/a11y.test.tsx b/src/plugins/AIComponents/__tests__/a11y.test.tsx index 56e232aa2..bee9b66fe 100644 --- a/src/plugins/AIComponents/__tests__/a11y.test.tsx +++ b/src/plugins/AIComponents/__tests__/a11y.test.tsx @@ -106,6 +106,7 @@ describe('AI components accessibility', () => { > + , ); expect(await axe(container)).toHaveNoViolations(); @@ -115,6 +116,9 @@ describe('AI components accessibility', () => { expect( screen.getByRole('img', { name: 'Voice agent: Thinking' }), ).toBeInTheDocument(); + expect( + screen.getByRole('img', { name: 'Voice agent: Speaking' }), + ).toBeInTheDocument(); }); it('announces the AI state politely', () => { diff --git a/src/plugins/AIComponents/__tests__/classNames.test.tsx b/src/plugins/AIComponents/__tests__/classNames.test.tsx index 8d84650f7..d415db9c8 100644 --- a/src/plugins/AIComponents/__tests__/classNames.test.tsx +++ b/src/plugins/AIComponents/__tests__/classNames.test.tsx @@ -22,6 +22,7 @@ it('emits no legacy aicr__ classes', () => { + , ); expect(container.innerHTML).not.toContain('aicr__'); @@ -30,4 +31,5 @@ it('emits no legacy aicr__ classes', () => { ).toBeInTheDocument(); expect(container.querySelector('.str-chat__ai-voice-wave')).toBeInTheDocument(); expect(container.querySelector('.str-chat__ai-voice-glow')).toBeInTheDocument(); + expect(container.querySelector('.str-chat__ai-voice-glow--css')).toBeInTheDocument(); }); diff --git a/src/plugins/AIComponents/__tests__/hydration.test.tsx b/src/plugins/AIComponents/__tests__/hydration.test.tsx index d41323d5d..5daea6d48 100644 --- a/src/plugins/AIComponents/__tests__/hydration.test.tsx +++ b/src/plugins/AIComponents/__tests__/hydration.test.tsx @@ -59,6 +59,7 @@ describe('VoiceWave and VoiceGlow hydration', () => { <> + ); const html = renderToString(); @@ -79,7 +80,8 @@ describe('VoiceWave and VoiceGlow hydration', () => { expect(container.querySelector('.str-chat__ai-voice-wave canvas')).toBe( serverCanvases[0], ); - expect(container.querySelectorAll('[role="img"]')).toHaveLength(2); + expect(container.querySelectorAll('[role="img"]')).toHaveLength(3); + expect(container.querySelector('.str-chat__ai-voice-glow--css')).toBeInTheDocument(); container.remove(); }); }); diff --git a/src/plugins/AIComponents/__tests__/ssr.test.tsx b/src/plugins/AIComponents/__tests__/ssr.test.tsx index 7360cb583..31d28383a 100644 --- a/src/plugins/AIComponents/__tests__/ssr.test.tsx +++ b/src/plugins/AIComponents/__tests__/ssr.test.tsx @@ -24,6 +24,7 @@ it('server-renders without touching browser globals', () => { + , ); expect(html).toContain('bold'); @@ -36,6 +37,7 @@ it('server-renders without touching browser globals', () => { expect(html).toContain( 'class="str-chat__ai-voice-visualization str-chat__ai-voice-glow"', ); - expect(html.match(/role="img"/g)).toHaveLength(2); + expect(html).toContain('str-chat__ai-voice-glow--css'); + expect(html.match(/role="img"/g)).toHaveLength(3); expect(html.match(/ { ); }); + it('animates the CSS voice glow only through transform and opacity', () => { + const keyframes = [ + ...css.matchAll(/@keyframes str-ai-glow-[a-z-]+\s*\{([\s\S]*?)\n\}/g), + ]; + expect(keyframes.length).toBeGreaterThan(10); + for (const [, body] of keyframes) { + const properties = [...body.matchAll(/([a-z-]+)\s*:/g)].map(([, name]) => name); + expect(properties.every((name) => name === 'transform' || name === 'opacity')).toBe( + true, + ); + } + }); + it('disables the state indicator animations for reduced motion', () => { const reduced = css.split('@media (prefers-reduced-motion: reduce)')[1] ?? ''; expect(reduced).toMatch( diff --git a/src/plugins/AIComponents/styling/index.scss b/src/plugins/AIComponents/styling/index.scss index f5d7e8cb2..d94180285 100644 --- a/src/plugins/AIComponents/styling/index.scss +++ b/src/plugins/AIComponents/styling/index.scss @@ -3,5 +3,6 @@ @use './ai-message-composer.scss'; @use './ai-state-indicator.scss'; @use './voice-visualization.scss'; +@use './voice-glow-css.scss'; // Intentionally declares no `--str-chat__ai-*` custom properties; see _tokens.scss. diff --git a/src/plugins/AIComponents/styling/voice-glow-css.scss b/src/plugins/AIComponents/styling/voice-glow-css.scss new file mode 100644 index 000000000..199f7ee7d --- /dev/null +++ b/src/plugins/AIComponents/styling/voice-glow-css.scss @@ -0,0 +1,552 @@ +// Intentionally declares no `--str-chat__ai-*` custom properties; see _tokens.scss. +@use './tokens' as *; + +// VoiceGlow with rendering='css': the shader's composition in CSS. An opaque color field (a +// base, the masses mixed over it in order, shade and rim) is seen through an alpha that only the +// first two masses make: a faint static body plus one mask riding each of their orbits, with the +// inverse turns inside each mask keeping the colors still. Only `transform` and `opacity` +// animate, so the compositor runs the motion. Sizes follow the shader: R = 0.24 of the shorter +// side, the soft edge at 1.06 R. +// +// The script sets the palette (--str-ai-glow-k0, -col-b to -col-e, -base, -halo, -rim, -shade), +// the falloff gradients and masks, the expressivity (--str-ai-glow-amp), each animation's +// playback rate for the state, and the live audio level. Until it runs, the masses take the +// theme's voice color. + +// Registered so a state change eases them, and every keyframe that reads them, instead of +// jumping. +@property --str-ai-glow-breath { + syntax: ''; + inherits: true; + initial-value: 0.015; +} + +@property --str-ai-glow-speech { + syntax: ''; + inherits: true; + initial-value: 0; +} + +@property --str-ai-glow-pulse { + syntax: ''; + inherits: true; + initial-value: 0; +} + +@property --str-ai-glow-sway { + syntax: ''; + inherits: true; + initial-value: 40deg; +} + +@property --str-ai-glow-warp { + syntax: ''; + inherits: true; + initial-value: 0.06; +} + +@property --str-ai-glow-halo-alpha { + syntax: ''; + inherits: true; + initial-value: 5.6%; +} + +// Written every frame while a live audio level is given; not inherited, so only the element +// that scales with it restyles. +@property --str-ai-glow-level { + syntax: ''; + inherits: false; + initial-value: 0; +} + +$glow: '.str-chat__ai-voice-glow'; +$ease: 0.6s ease-out; + +#{$glow}--css { + --str-ai-glow-k0: #{ai-token(voice-color)}; + --str-ai-glow-col-b: var(--str-ai-glow-k0); + --str-ai-glow-col-c: var(--str-ai-glow-k0); + --str-ai-glow-col-d: var(--str-ai-glow-k0); + --str-ai-glow-col-e: var(--str-ai-glow-k0); + --str-ai-glow-base: var(--str-ai-glow-k0); + --str-ai-glow-halo: var(--str-ai-glow-k0); + --str-ai-glow-rim: var(--str-ai-glow-k0); + + display: grid; + place-items: center; + container-type: size; + transition: + --str-ai-glow-breath $ease, + --str-ai-glow-speech $ease, + --str-ai-glow-pulse $ease, + --str-ai-glow-sway $ease, + --str-ai-glow-warp $ease, + --str-ai-glow-halo-alpha $ease; + + // How far each state moves; how fast is the script's playback rates. + &[data-state='idle'] { + --str-ai-glow-breath: 0.015; + --str-ai-glow-speech: 0; + --str-ai-glow-pulse: 0; + --str-ai-glow-sway: 40deg; + --str-ai-glow-warp: 0.06; + --str-ai-glow-halo-alpha: 5.6%; + } + + &[data-state='connecting'] { + --str-ai-glow-breath: 0.05; + --str-ai-glow-speech: 0; + --str-ai-glow-pulse: 1; + --str-ai-glow-sway: 50deg; + --str-ai-glow-warp: 0.1; + --str-ai-glow-halo-alpha: 5%; + } + + &[data-state='listening'] { + --str-ai-glow-breath: 0.04; + --str-ai-glow-speech: 0.35; + --str-ai-glow-pulse: 0; + --str-ai-glow-sway: 45deg; + --str-ai-glow-warp: 0.1; + --str-ai-glow-halo-alpha: 7.3%; + } + + &[data-state='thinking'] { + --str-ai-glow-breath: 0.02; + --str-ai-glow-speech: 0; + --str-ai-glow-pulse: 0; + --str-ai-glow-sway: 60deg; + --str-ai-glow-warp: 0.12; + --str-ai-glow-halo-alpha: 5%; + } + + &[data-state='speaking'] { + --str-ai-glow-breath: 0.04; + --str-ai-glow-speech: 1; + --str-ai-glow-pulse: 0; + --str-ai-glow-sway: 45deg; + --str-ai-glow-warp: 0.14; + --str-ai-glow-halo-alpha: 10%; + } + + &[data-state][data-dark='true'] { + --str-ai-glow-halo-alpha: 12%; + } + + // A live audio level replaces the self-animated speech, as in the other renderers. + &[data-state][data-live='true'] { + --str-ai-glow-speech: 0; + } +} + +#{$glow}__halo, +#{$glow}__level { + grid-area: 1 / 1; +} + +// Behind the globe: solid to 0.8 R, e^-1 a further 0.09 of the side, gone at 0.8 R + 0.25 side. +#{$glow}__halo { + width: 88.4cqmin; + height: 88.4cqmin; + border-radius: 50%; + background: radial-gradient( + closest-side, + color-mix(in srgb, var(--str-ai-glow-halo) var(--str-ai-glow-halo-alpha), transparent) + 43%, + color-mix( + in srgb, + var(--str-ai-glow-halo) calc(var(--str-ai-glow-halo-alpha) * 0.37), + transparent + ) + 63%, + transparent 100% + ); + animation: str-ai-glow-breath 3.9s ease-in-out infinite alternate; +} + +#{$glow}__level { + position: relative; + width: 50.9cqmin; + height: 50.9cqmin; + transform: scale(calc(1 + var(--str-ai-glow-level) * 0.2)); +} + +#{$glow}__bounce, +#{$glow}__breath, +#{$glow}__gate, +#{$glow}__speech, +#{$glow}__pulse, +#{$glow}__globe, +#{$glow}__body, +#{$glow}__field, +#{$glow}__base, +#{$glow}__flow, +#{$glow}__unflow, +#{$glow}__orbit, +#{$glow}__unorbit, +#{$glow}__sway, +#{$glow}__unsway, +#{$glow}__warp, +#{$glow}__unwarp, +#{$glow}__shade, +#{$glow}__rim { + position: absolute; + inset: 0; +} + +// The slow expand and contract every state has. +#{$glow}__breath { + animation: str-ai-glow-breath 3.9s ease-in-out infinite alternate; +} + +@keyframes str-ai-glow-breath { + from { + transform: scale( + calc(1 + var(--str-ai-glow-breath) * var(--str-ai-glow-amp, 1) * 0.2) + ); + } + + to { + transform: scale(calc(1 + var(--str-ai-glow-breath) * var(--str-ai-glow-amp, 1))); + } +} + +// Speech: swells of mismatched lengths, so together they ebb and pause like words rather than +// loop. Their size is --str-ai-glow-speech, so other states keep them flat. +#{$glow}__speech { + animation: str-ai-glow-syllables 0.83s ease-in-out infinite alternate; +} + +#{$glow}__gate { + animation: str-ai-glow-phrases 2.9s ease-in-out infinite; +} + +@keyframes str-ai-glow-syllables { + from { + transform: scale( + calc(1 + var(--str-ai-glow-speech) * var(--str-ai-glow-amp, 1) * 0.02) + ); + } + + to { + transform: scale( + calc(1 + var(--str-ai-glow-speech) * var(--str-ai-glow-amp, 1) * 0.12) + ); + } +} + +@keyframes str-ai-glow-phrases { + 0%, + 100% { + transform: scale(1); + } + + 20% { + transform: scale( + calc(1 + var(--str-ai-glow-speech) * var(--str-ai-glow-amp, 1) * 0.08) + ); + } + + 45% { + transform: scale( + calc(1 + var(--str-ai-glow-speech) * var(--str-ai-glow-amp, 1) * 0.02) + ); + } + + 70% { + transform: scale( + calc(1 + var(--str-ai-glow-speech) * var(--str-ai-glow-amp, 1) * 0.1) + ); + } +} + +// Connecting: the glow dims and brightens, as 0.45 + 0.35 sin(2t) in the shader. +#{$glow}__pulse { + animation: str-ai-glow-pulse 3.14s ease-in-out infinite alternate; +} + +@keyframes str-ai-glow-pulse { + from { + opacity: calc(1 - var(--str-ai-glow-pulse) * 0.9); + } + + to { + opacity: calc(1 - var(--str-ai-glow-pulse) * 0.2); + } +} + +// The globe: body = smoothstep(1.06, 0.62, r / R), as a static mask. Masks only read alpha, so +// they are drawn in the root's (opaque) color. +#{$glow}__globe { + border-radius: 50%; + overflow: hidden; + isolation: isolate; + -webkit-mask-image: radial-gradient( + closest-side, + currentColor 58.5%, + color-mix(in srgb, currentColor 84%, transparent) 69%, + color-mix(in srgb, currentColor 50%, transparent) 79%, + color-mix(in srgb, currentColor 16%, transparent) 89%, + transparent 100% + ); + mask-image: radial-gradient( + closest-side, + currentColor 58.5%, + color-mix(in srgb, currentColor 84%, transparent) 69%, + color-mix(in srgb, currentColor 50%, transparent) 79%, + color-mix(in srgb, currentColor 16%, transparent) 89%, + transparent 100% + ); +} + +// The faint body everywhere: 0.15, a touch more at the core. +#{$glow}__body--still { + -webkit-mask-image: radial-gradient( + closest-side, + color-mix(in srgb, currentColor 21%, transparent), + color-mix(in srgb, currentColor 15%, transparent) + ); + mask-image: radial-gradient( + closest-side, + color-mix(in srgb, currentColor 21%, transparent), + color-mix(in srgb, currentColor 15%, transparent) + ); +} + +// Thinking turns the whole field of color slowly; other states play it at rate 0. +#{$glow}__flow { + animation: str-ai-glow-flow 16.8s linear infinite; +} + +#{$glow}__unflow { + animation: str-ai-glow-unflow 16.8s linear infinite; +} + +@keyframes str-ai-glow-flow { + to { + transform: rotate(360deg); + } +} + +@keyframes str-ai-glow-unflow { + to { + transform: rotate(-360deg); + } +} + +// Each mass orbits the centre at its own pace and direction, swaying on top in place of the +// noise that wobbles its orbit. The un- versions are exact inverses, run in step. +#{$glow}__orbit { + animation: str-ai-glow-spin var(--str-ai-glow-period) linear infinite + var(--str-ai-glow-direction, normal); +} + +#{$glow}__unorbit { + animation: str-ai-glow-unspin var(--str-ai-glow-period) linear infinite + var(--str-ai-glow-direction, normal); +} + +#{$glow}__sway { + animation: str-ai-glow-sway var(--str-ai-glow-sway-period) ease-in-out infinite + alternate; +} + +#{$glow}__unsway { + animation: str-ai-glow-unsway var(--str-ai-glow-sway-period) ease-in-out infinite + alternate; +} + +@keyframes str-ai-glow-spin { + from { + transform: rotate(var(--str-ai-glow-start)); + } + + to { + transform: rotate(calc(var(--str-ai-glow-start) + 360deg)); + } +} + +@keyframes str-ai-glow-unspin { + from { + transform: rotate(calc(var(--str-ai-glow-start) * -1)); + } + + to { + transform: rotate(calc(var(--str-ai-glow-start) * -1 - 360deg)); + } +} + +@keyframes str-ai-glow-sway { + from { + transform: rotate(calc(var(--str-ai-glow-sway) * var(--str-ai-glow-amp, 1) * -1)); + } + + to { + transform: rotate(calc(var(--str-ai-glow-sway) * var(--str-ai-glow-amp, 1))); + } +} + +@keyframes str-ai-glow-unsway { + from { + transform: rotate(calc(var(--str-ai-glow-sway) * var(--str-ai-glow-amp, 1))); + } + + to { + transform: rotate(calc(var(--str-ai-glow-sway) * var(--str-ai-glow-amp, 1) * -1)); + } +} + +// The shader's noise bends the masses; a slow, uneven stretch of each stands in for it. The +// riding masks stretch twice as far, and the inverse stretch inside them keeps the colors still. +#{$glow}__warp, +#{$glow}__unwarp { + transform-origin: calc(50% + var(--str-ai-glow-dist)) 50%; +} + +#{$glow}__warp { + animation: str-ai-glow-warp var(--str-ai-glow-warp-period) ease-in-out infinite + alternate; +} + +#{$glow}__warp#{$glow}__warp--mask { + -webkit-mask-image: var(--str-ai-glow-mask); + mask-image: var(--str-ai-glow-mask); + animation-name: str-ai-glow-mask-warp; +} + +#{$glow}__unwarp { + animation: str-ai-glow-mask-unwarp var(--str-ai-glow-warp-period) ease-in-out infinite + alternate; +} + +@mixin stretch($amount) { + transform: rotate(var(--str-ai-glow-warp-turn)) + scale(#{nth($amount, 1)}, #{nth($amount, 2)}) + rotate(calc(var(--str-ai-glow-warp-turn) * -1)); +} + +$w: 'var(--str-ai-glow-warp) * var(--str-ai-glow-amp, 1)'; + +@keyframes str-ai-glow-warp { + from { + @include stretch((calc(1 + #{$w}), calc(1 - #{$w}))); + } + + to { + @include stretch((calc(1 - #{$w}), calc(1 + #{$w}))); + } +} + +@keyframes str-ai-glow-mask-warp { + from { + @include stretch((calc(1 + #{$w} * 2), calc(1 - #{$w} * 2))); + } + + to { + @include stretch((calc(1 - #{$w} * 2), calc(1 + #{$w} * 2))); + } +} + +@keyframes str-ai-glow-mask-unwarp { + from { + @include stretch((calc(1 / (1 + #{$w} * 2)), calc(1 / (1 - #{$w} * 2)))); + } + + to { + @include stretch((calc(1 / (1 - #{$w} * 2)), calc(1 / (1 + #{$w} * 2)))); + } +} + +#{$glow}__field { + isolation: isolate; +} + +#{$glow}__base { + background: var(--str-ai-glow-base); +} + +// A mass: centred 0.4 R from the middle, reaching its reach in R. Its falloff comes from the +// script. +#{$glow}__mass { + position: absolute; + top: 50%; + left: calc(50% + var(--str-ai-glow-dist)); + width: calc(var(--str-ai-glow-reach) * 94.34%); + aspect-ratio: 1; + border-radius: 50%; + translate: -50% -50%; +} + +// Diffuse light from the upper left: 1 where the globe faces it, 0.8 on the far side. +#{$glow}__shade { + background: radial-gradient( + circle at 33% 27%, + transparent 15%, + color-mix(in srgb, var(--str-ai-glow-shade) 8%, transparent) 55%, + color-mix(in srgb, var(--str-ai-glow-shade) 20%, transparent) 100% + ); +} + +// Fresnel: light gathering at the rim, added on top. +#{$glow}__rim { + background: radial-gradient( + closest-side, + transparent 55%, + color-mix(in srgb, var(--str-ai-glow-rim) 4%, transparent) 80%, + color-mix(in srgb, var(--str-ai-glow-rim) 10%, transparent) 100% + ); + mix-blend-mode: plus-lighter; +} + +// A tap: the same dip and overshoot as the other renderers, and a ring in place of the ripple. +#{$glow}__bounce--active { + animation: str-ai-glow-bounce 0.34s ease-out; +} + +@keyframes str-ai-glow-bounce { + 0% { + transform: scale(1); + } + + 30% { + transform: scale(0.93); + } + + 62% { + transform: scale(1.03); + } + + 100% { + transform: scale(1); + } +} + +#{$glow}__ripple { + position: absolute; + width: 20cqmin; + aspect-ratio: 1; + border-radius: 50%; + translate: -50% -50%; + border: 2px solid color-mix(in srgb, var(--str-ai-glow-rim) 60%, transparent); + pointer-events: none; + animation: str-ai-glow-ripple 0.9s ease-out forwards; +} + +@keyframes str-ai-glow-ripple { + from { + transform: scale(0.2); + opacity: 0.8; + } + + to { + transform: scale(2.4); + opacity: 0; + } +} + +@media (prefers-reduced-motion: reduce) { + #{$glow}__ripple, + #{$glow}__bounce--active { + animation: none; + } +} From dc7695c85105e6cac335fc7dd7a1d17f22848b49 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 21 Oct 2026 13:38:59 +0200 Subject: [PATCH 4/8] fix(ai-components): match the iOS voice visualizations on pass-through, speech rhythm and fade-in - Only the shape takes the pointer: the root lets events through to whatever is behind it, and a hit area the size of the globe or the string takes them and hands them to the root's handlers. - The wave and the Canvas glow animate speech with the shader's simplex envelope, ported in float32, instead of a 1-D CPU noise, so every renderer and iOS share the same rhythm. - The visualizations fade in over 0.3s once their first frame is drawn (the CSS glow once its palette is set), instantly with reduced motion. --- src/plugins/AIComponents/README.md | 3 +- .../VoiceVisualization/VoiceGlowCss.tsx | 4 + .../VoiceVisualizationView.tsx | 8 +- .../AIComponents/VoiceVisualization/math.ts | 10 ++- .../VoiceVisualization/simplex.ts | 84 +++++++++++++++++++ .../__tests__/VoiceGlowCss.test.tsx | 9 ++ .../__tests__/VoiceVisualization.test.tsx | 52 ++++++++++++ .../__tests__/styleTokens.test.ts | 22 +++++ .../__tests__/voicePalette.test.ts | 33 +++++++- .../AIComponents/styling/voice-glow-css.scss | 12 +++ .../styling/voice-visualization.scss | 40 +++++++++ 11 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 src/plugins/AIComponents/VoiceVisualization/simplex.ts diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md index 126b8cc5f..635c56c81 100644 --- a/src/plugins/AIComponents/README.md +++ b/src/plugins/AIComponents/README.md @@ -316,11 +316,12 @@ Both draw on a `` inside a `div` with `role="img"` and an accessible nam | `audioLevel` | `() => number \| null \| undefined` | no | The level of whoever is talking, from 0 to 1, read once a frame, so a level changing many times a second re-renders nothing. Returning `null` (or omitting it) animates speech on its own. | | `rendering` | `'auto' \| 'webgl' \| 'canvas' \| 'css'` | no | `VoiceGlow` only. `auto` (default) and `webgl` draw with a WebGL shader and fall back to a 2D canvas when WebGL is unavailable or its context is lost; `canvas` always uses the cheaper 2D drawing; `css` builds the glow from CSS gradients animated by the browser's compositor (see below). | -Any other `div` prop (`className`, `style`, event handlers, ...) is passed to the root. Pointer events are watched, never stopped: a tap inside the shape bounces it and a hovering mouse or pen ripples it, while your own `onClick` still fires. +Any other `div` prop (`className`, `style`, event handlers, ...) is passed to the root. Only the shape takes the pointer, as on iOS: the globe (within 0.32 of the shorter side from the centre) or the string. Elsewhere in the box, clicks and hovers reach whatever is behind it. On the shape, pointer events are watched and never stopped: a tap bounces it and a hovering mouse or pen ripples it, while your own `onClick` on the root or a parent still fires. Behavior: - The animation runs only while the element is on screen and the tab is visible. +- It fades in over 0.3 s once its first frame is drawn (instantly with reduced motion), so it never flashes empty while a shader compiles. - With `prefers-reduced-motion: reduce`, the motion slows to a crawl and taps and hover are ignored. - The palette and the light or dark look follow the theme (`--str-chat__ai-voice-color` and `--str-chat__ai-voice-backdrop`), re-read about once a second, so a theme switch needs no prop. diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx index 70d2d7fe3..e984006f7 100644 --- a/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx @@ -188,6 +188,8 @@ export const VoiceGlowCss = ({ root.style.setProperty(name, value); } root.setAttribute('data-dark', String(!!dark)); + // Fades in once the palette is in, so the theme color it starts from never shows. + root.setAttribute('data-ready', 'true'); }; apply(); const interval = setInterval(apply, THEME_INTERVAL); @@ -316,6 +318,8 @@ export const VoiceGlowCss = ({
+ {/* Only the globe takes the pointer; around it, events reach what is behind. */} +