feat(voice): add adjustable lip-sync controls (#67)

* feat(voice): add adjustable lip-sync controls

* test(voice): keep lip-sync normalization test pure

* fix(voice): stack lip-sync sliders and add section dividers

Match Audio source field-label layout (title → control → hint) and
Settings-style panel dividers between Voice sections.

---------

Co-authored-by: rosspeili <vpeilivanidis@gmail.com>

Author: AUDOSt0ck1ng

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Added
 
+- **Adjustable lip-sync response** — Gear → **Voice** now exposes **Sensitivity** (`0.25`–`4.00`) for quiet inputs and **Mouth limit** (`0.10`–`1.00`) for the VRM expression ceiling. Both values persist in `config.yaml`, default to the previous response at `1.00`, and reset on double-click.
 - **SECURITY.md** — supported versions, private reporting (`input@arpacorp.net` or GitHub Security Advisories), in-scope areas (installer, loopback agent bus/MCP, capture, VRoid OAuth, persistence), and out-of-scope boundaries. Linked from README and CONTRIBUTING. (#26)
 - **Asset license manifest** — [`docs/assets-manifest.yml`](docs/assets-manifest.yml) inventories bundled VRM/VRMA/environment media, derived thumbnails, installer branding, documentation screenshots, and runtime VRM libraries with paths, licenses, credit lines, and audit status. Validated in `npm test` via `avatar/scripts/validate-assets-manifest.mjs`. Linked from [Assets & credits](docs/assets-and-credits.md), README, CONTRIBUTING, and the maintainer release checklist. (#11)
 - **Local agent bus** — Settings → **Agents**. An opt-in loopback server (`127.0.0.1:47903`, off by default) so scripts and agent frameworks can drive the avatar: `POST /v1/command` and a WebSocket at `/v1/socket` as peers, both dispatching the *existing* stage commands (`animation.play`, `animation.default`, `animation.stop`, `avatar.set`, `environment.set`, `audio.source`) rather than a second set of names. `GET /v1/state` lists what is on stage with **id and label**, since a custom folder derives animation ids from file paths and a caller cannot invent them — the label works as a play id, and `playableOnce` says which clips accept `"mode": "once"`. Omitting `mode` still means *select*, which persists to `config.yaml`; agents almost always want `once`, and every example says so. Validation happens in the Electron main process against a catalog the window reports, so a request is answered with the same error codes the UI produces and the accepted action is applied by the window — a `200` means accepted, not that the model finished loading. A token is minted on first enable, reused after that, and stored encrypted with the OS keychain rather than in `config.yaml` (which the renderer rewrites on every change); it travels in `Authorization: Bearer`, never a query string. Anything carrying an `Origin` header is refused on both transports, so a web page — including a local dev server — cannot drive the avatar behind your back. Fixed port with no silent fallback, because the copied `curl` example names one. The WebSocket replies to what it is sent and pushes nothing; every reply carries an `id`, so events could be added later as frames without one. MCP is deliberately not in this: an adapter can sit on top of these HTTP commands later — which is what `/mcp` below now does. See [Local agent bus](docs/agents/local-bus.md). (#6)

avatar/src/components/AvatarStage.jsx

@@ -6,6 +6,10 @@ import {
   resolveAnimationId,
 } from '../config/animations';
 import { getAudioSourceOptions, getDefaultAudioSourceId } from '../config/audioSources';
+import {
+  defaultLipSyncMouthStrength,
+  defaultLipSyncSensitivity,
+} from '../config/lipSyncSensitivity';
 import {
   customEnvironments,
   defaultColor,
@@ -90,6 +94,8 @@ export function AvatarStage() {
   const [audioFile, setAudioFile] = useState(null);
   const [windowSourceId, setWindowSourceId] = useState(null);
   const [audioSourceId, setAudioSourceId] = useState(getDefaultAudioSourceId);
+  const [lipSyncSensitivity, setLipSyncSensitivity] = useState(defaultLipSyncSensitivity);
+  const [lipSyncMouthStrength, setLipSyncMouthStrength] = useState(defaultLipSyncMouthStrength);
   const [settingsPath, setSettingsPath] = useState(null);
   // Session-only: a VRoid Hub character is never persisted to config.yaml
   // (kept in memory only, per VRoid Hub's licensing rules for linked-app
@@ -126,7 +132,7 @@ export function AvatarStage() {
   const { level, levelRef, speaking, status: audioStatus, error: audioError, restart } = useAudioSource(
     audioSourceId,
     audioFile,
-    { windowSourceId },
+    { windowSourceId, sensitivity: lipSyncSensitivity },
   );
 
   const lipSyncEnabled = audioSourceId !== 'none' && audioStatus === 'active';
@@ -156,6 +162,8 @@ export function AvatarStage() {
       rotation: [...settings.avatarTransform.rotation],
     });
     setAudioSourceId(settings.audioSourceId);
+    setLipSyncSensitivity(settings.lipSyncSensitivity);
+    setLipSyncMouthStrength(settings.lipSyncMouthStrength);
     setWindowSourceId(settings.windowSourceId);
     setOverlayMode(settings.overlayMode);
     document.documentElement.classList.toggle('vox-desktop-windowed', !settings.overlayMode);
@@ -241,6 +249,8 @@ export function AvatarStage() {
         light,
         avatarTransform: avatar,
         audioSourceId,
+        lipSyncSensitivity,
+        lipSyncMouthStrength,
         windowSourceId,
         overlayMode,
         windowScale,
@@ -262,6 +272,8 @@ export function AvatarStage() {
     light,
     avatar,
     audioSourceId,
+    lipSyncSensitivity,
+    lipSyncMouthStrength,
     windowSourceId,
     overlayMode,
     windowScale,
@@ -1022,6 +1034,7 @@ export function AvatarStage() {
               avatarRotation={avatar.rotation}
               audioLevel={level}
               lipSyncEnabled={lipSyncEnabled}
+              lipSyncMouthStrength={lipSyncMouthStrength}
               speaking={speaking}
               onLoaded={handleAvatarLoaded}
             />
@@ -1067,6 +1080,10 @@ export function AvatarStage() {
               audioStatus={audioStatus}
               audioError={audioError}
               onRestartAudio={() => void restart()}
+              lipSyncSensitivity={lipSyncSensitivity}
+              onLipSyncSensitivityChange={setLipSyncSensitivity}
+              lipSyncMouthStrength={lipSyncMouthStrength}
+              onLipSyncMouthStrengthChange={setLipSyncMouthStrength}
             />
           )}
 

avatar/src/components/avatar/VrmAvatar.jsx

@@ -27,6 +27,7 @@ export function VrmAvatar({
   avatarRotation,
   audioLevel = 0,
   lipSyncEnabled = false,
+  lipSyncMouthStrength = 1,
   speaking = false,
   onLoaded,
 }) {
@@ -41,7 +42,7 @@ export function VrmAvatar({
   const overlayToken = motionOverlay?.token ?? 0;
 
   const { play, playSequence, cancel, returnToRest, update: updateMixer } = useVrmAnimation(vrm);
-  const updateLipSync = useAmplitudeLipSync(vrm);
+  const updateLipSync = useAmplitudeLipSync(vrm, lipSyncMouthStrength);
   const updateBlink = useBlink(vrm);
 
   useEffect(() => {

avatar/src/components/panels/VoicePanel.jsx

@@ -5,7 +5,11 @@ import {
   labelAudioCaptureStatus,
 } from '../../lib/audioCaptureCopy';
 import { getDesktopApi } from '../../lib/desktopMode';
-import { PanelSelect } from '../ui/PanelPrimitives';
+import {
+  defaultLipSyncMouthStrength,
+  defaultLipSyncSensitivity,
+} from '../../config/lipSyncSensitivity';
+import { PanelSelect, SliderRow, Divider } from '../ui/PanelPrimitives';
 
 export function VoicePanel({
   audioSourceId,
@@ -16,6 +20,10 @@ export function VoicePanel({
   audioStatus,
   audioError,
   onRestartAudio,
+  lipSyncSensitivity,
+  onLipSyncSensitivityChange,
+  lipSyncMouthStrength,
+  onLipSyncMouthStrengthChange,
 }) {
   const [windowSources, setWindowSources] = useState([]);
   const audioSourceOptions = getAudioSourceOptions();
@@ -63,8 +71,48 @@ export function VoicePanel({
         {audioSourceOptions.find((option) => option.id === audioSourceId)?.description}
       </p>
 
+      {audioSourceId !== 'none' && (
+        <>
+          <Divider />
+
+          <SliderRow
+            stacked
+            id="lip-sync-sensitivity"
+            label="Sensitivity"
+            min={0.25}
+            max={4}
+            step={0.25}
+            value={lipSyncSensitivity}
+            onChange={onLipSyncSensitivityChange}
+            onDoubleClick={() => onLipSyncSensitivityChange(defaultLipSyncSensitivity)}
+          />
+          <p className="panel-hint">
+            Raise this when audio is detected but the mouth barely moves. Double-click to reset.
+          </p>
+
+          <Divider />
+
+          <SliderRow
+            stacked
+            id="lip-sync-mouth-limit"
+            label="Mouth limit"
+            min={0.1}
+            max={1}
+            step={0.05}
+            value={lipSyncMouthStrength}
+            onChange={onLipSyncMouthStrengthChange}
+            onDoubleClick={() => onLipSyncMouthStrengthChange(defaultLipSyncMouthStrength)}
+          />
+          <p className="panel-hint">
+            Limits the maximum VRM mouth-expression weight. Double-click to reset.
+          </p>
+        </>
+      )}
+
       {audioSourceId === 'window' && (
         <>
+          <Divider />
+
           <label className="field-label" htmlFor="window-source-select">
             Window or screen
           </label>
@@ -79,15 +127,20 @@ export function VoicePanel({
       )}
 
       {audioSourceId === 'file' && (
-        <div className="file-picker-row">
-          <input
-            type="file"
-            accept="audio/*"
-            onChange={(event) => setAudioFile(event.target.files?.[0] ?? null)}
-          />
-        </div>
+        <>
+          <Divider />
+          <div className="file-picker-row">
+            <input
+              type="file"
+              accept="audio/*"
+              onChange={(event) => setAudioFile(event.target.files?.[0] ?? null)}
+            />
+          </div>
+        </>
       )}
 
+      <Divider />
+
       <div className="voice-status-row">
         <span
           className={`voice-status voice-status--${audioStatus}`}

avatar/src/components/ui/PanelPrimitives.jsx

@@ -18,11 +18,24 @@ export function Divider() {
   return <div className="panel-divider" />;
 }
 
-export function SliderRow({ label, min, max, step, value, onChange, onDoubleClick }) {
-  return (
-    <div className="slider-row">
-      <span className="slider-row__label">{label}</span>
+export function SliderRow({
+  label,
+  id,
+  min,
+  max,
+  step,
+  value,
+  onChange,
+  onDoubleClick,
+  /** Stacked: field-label above, slider+value below (Voice / form fields). Default is inline (Camera). */
+  stacked = false,
+}) {
+  const inputId = id ?? undefined;
+  const control = (
+    <div className={`slider-row${stacked ? ' slider-row--control' : ''}`}>
+      {!stacked && <span className="slider-row__label">{label}</span>}
       <input
+        id={inputId}
         type="range"
         min={min}
         max={max}
@@ -31,10 +44,22 @@ export function SliderRow({ label, min, max, step, value, onChange, onDoubleClic
         onChange={(event) => onChange(parseFloat(event.target.value))}
         onDoubleClick={onDoubleClick}
         className="slider-row__input"
+        aria-label={stacked ? undefined : label}
       />
       <span className="slider-row__value">{typeof value === 'number' ? value.toFixed(2) : value}</span>
     </div>
   );
+
+  if (!stacked) return control;
+
+  return (
+    <>
+      <label className="field-label" htmlFor={inputId}>
+        {label}
+      </label>
+      {control}
+    </>
+  );
 }
 
 const SELECT_MENU_MAX_HEIGHT = 148;

avatar/src/config/lipSyncSensitivity.js

@@ -0,0 +1,22 @@
+export const defaultLipSyncSensitivity = 1;
+export const minLipSyncSensitivity = 0.25;
+export const maxLipSyncSensitivity = 4;
+export const defaultLipSyncMouthStrength = 1;
+export const minLipSyncMouthStrength = 0.1;
+export const maxLipSyncMouthStrength = 1;
+
+/** @param {unknown} value */
+export function normalizeLipSyncSensitivity(value) {
+  if (value == null) return defaultLipSyncSensitivity;
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric)) return defaultLipSyncSensitivity;
+  return Math.min(maxLipSyncSensitivity, Math.max(minLipSyncSensitivity, numeric));
+}
+
+/** @param {unknown} value */
+export function normalizeLipSyncMouthStrength(value) {
+  if (value == null) return defaultLipSyncMouthStrength;
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric)) return defaultLipSyncMouthStrength;
+  return Math.min(maxLipSyncMouthStrength, Math.max(minLipSyncMouthStrength, numeric));
+}

avatar/src/config/lipSyncSensitivity.test.mjs

@@ -0,0 +1,31 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  defaultLipSyncMouthStrength,
+  defaultLipSyncSensitivity,
+  normalizeLipSyncMouthStrength,
+  normalizeLipSyncSensitivity,
+} from './lipSyncSensitivity.js';
+
+test('normalizeLipSyncSensitivity keeps supported values', () => {
+  assert.equal(normalizeLipSyncSensitivity(0.25), 0.25);
+  assert.equal(normalizeLipSyncSensitivity(1), 1);
+  assert.equal(normalizeLipSyncSensitivity('2.5'), 2.5);
+  assert.equal(normalizeLipSyncSensitivity(4), 4);
+});
+
+test('normalizeLipSyncMouthStrength clamps to the VRM expression range', () => {
+  assert.equal(normalizeLipSyncMouthStrength(0), 0.1);
+  assert.equal(normalizeLipSyncMouthStrength(0.65), 0.65);
+  assert.equal(normalizeLipSyncMouthStrength(2), 1);
+  assert.equal(normalizeLipSyncMouthStrength(null), defaultLipSyncMouthStrength);
+  assert.equal(normalizeLipSyncMouthStrength(undefined), defaultLipSyncMouthStrength);
+});
+
+test('normalizeLipSyncSensitivity clamps or defaults invalid values', () => {
+  assert.equal(normalizeLipSyncSensitivity(0), 0.25);
+  assert.equal(normalizeLipSyncSensitivity(10), 4);
+  assert.equal(normalizeLipSyncSensitivity('nope'), defaultLipSyncSensitivity);
+  assert.equal(normalizeLipSyncSensitivity(null), defaultLipSyncSensitivity);
+  assert.equal(normalizeLipSyncSensitivity(undefined), defaultLipSyncSensitivity);
+});

avatar/src/config/userSettings.js

@@ -6,6 +6,12 @@ import { defaultAvatar, defaultCamera, defaultLight } from './defaults';
 import { defaultColor, normalizeEnvironmentSelection } from './environmentSelection';
 import { defaultWindowScale, normalizeWindowScale } from './windowScale';
 import { createEmptyDeck, normalizeMotionDeck } from '../motion-deck/motionDeck';
+import {
+  defaultLipSyncMouthStrength,
+  defaultLipSyncSensitivity,
+  normalizeLipSyncMouthStrength,
+  normalizeLipSyncSensitivity,
+} from './lipSyncSensitivity';
 
 // 2: avatarTransform.rotation is the user's framing rotation only. Version 1
 // seeded it from a default that had the VRM 0.0 180° facing flip baked in, so
@@ -53,6 +59,8 @@ export function createDefaultUserSettings() {
       rotation: [...defaultAvatar.rotation],
     },
     audioSourceId: getDefaultAudioSourceId(),
+    lipSyncSensitivity: defaultLipSyncSensitivity,
+    lipSyncMouthStrength: defaultLipSyncMouthStrength,
     windowSourceId: null,
     overlayMode: true,
     windowScale: defaultWindowScale,
@@ -198,6 +206,8 @@ export function normalizeUserSettings(raw) {
       ),
     },
     audioSourceId: asString(data.audioSourceId, defaults.audioSourceId),
+    lipSyncSensitivity: normalizeLipSyncSensitivity(data.lipSyncSensitivity),
+    lipSyncMouthStrength: normalizeLipSyncMouthStrength(data.lipSyncMouthStrength),
     windowSourceId: typeof data.windowSourceId === 'string' ? data.windowSourceId : null,
     overlayMode: data.overlayMode !== false,
     windowScale: normalizeWindowScale(asNumber(data.windowScale, defaults.windowScale)),
@@ -227,6 +237,8 @@ export function snapshotUserSettings(state) {
     light: state.light,
     avatarTransform: state.avatarTransform,
     audioSourceId: state.audioSourceId,
+    lipSyncSensitivity: state.lipSyncSensitivity,
+    lipSyncMouthStrength: state.lipSyncMouthStrength,
     windowSourceId: state.windowSourceId ?? null,
     overlayMode: state.overlayMode,
     windowScale: state.windowScale,

avatar/src/hooks/useAmplitudeLipSync.js

@@ -8,7 +8,7 @@ const VISEMES = ['aa', 'ee', 'ih', 'oh', 'ou'];
  *
  * @param {import('@pixiv/three-vrm').VRM | null} vrm
  */
-export function useAmplitudeLipSync(vrm) {
+export function useAmplitudeLipSync(vrm, mouthStrength = 1) {
   const smoothed = useRef(0);
   const phase = useRef(0);
 
@@ -27,10 +27,13 @@ export function useAmplitudeLipSync(vrm) {
       for (let index = 0; index < VISEMES.length; index += 1) {
         const shape = Math.max(0, 1 - Math.abs(index - active) * 0.72);
         const flutter = 0.74 + Math.sin(phase.current * 5.7 + index) * 0.18;
-        manager.setValue(VISEMES[index], Math.min(0.62, smoothed.current * shape * flutter));
+        manager.setValue(
+          VISEMES[index],
+          Math.min(mouthStrength, smoothed.current * shape * flutter),
+        );
       }
     },
-    [vrm],
+    [mouthStrength, vrm],
   );
 }