Add voice recording, transcription, and audio player components

- Implemented `AudioPlayer` for custom audio playback controls.
- Added `useAudioRecorder` hook for managing microphone audio recording with visual feedback.
- Introduced audio transcription API integration using OpenAI Whisper.
- Created `blobToBase64` utility to encode recorded blobs for API consumption.
- Added IndexedDB-based `recording-store` for saving in-progress recordings.
- Developed `VoiceAutofillSection` for guided recording, playback, transcription, and editing.

Author: MartinBraquet

android/app/build.gradle

@@ -11,8 +11,8 @@ android {
         applicationId "com.compassconnections.app"
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
-        versionCode 131
-        versionName "1.31.0"
+        versionCode 132
+        versionName "1.32.0"
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         aaptOptions {
              // Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps.

backend/api/src/app.ts

@@ -92,6 +92,7 @@ import {setLastOnlineTime} from './set-last-online-time'
 import {shipProfiles} from './ship-profiles'
 import {starProfile} from './star-profile'
 import {stats} from './stats'
+import {transcribeAudio} from './transcribe-audio'
 import {unsubscribe} from './unsubscribe'
 import {updateEvent} from './update-event'
 import {updateMe} from './update-me'
@@ -653,6 +654,7 @@ const handlers: {[k in APIPath]: APIHandler<k>} = {
   vote: vote,
   'validate-username': validateUsernameEndpoint,
   'llm-extract-profile': llmExtractProfileEndpoint,
+  'transcribe-audio': transcribeAudio,
   // 'user/:username': getUser,
   // 'user/:username/lite': getDisplayUser,
   // 'user/by-id/:id/lite': getDisplayUser,
@@ -678,7 +680,8 @@ Object.entries(handlers).forEach(([path, handler]) => {
 
   const apiRoute = [
     url,
-    express.json({limit: '1mb'}),
+    // Endpoints that carry binary payloads (e.g. base64 audio) declare a larger `bodyLimit`.
+    express.json({limit: (api as any).bodyLimit ?? '1mb'}),
     allowCorsUnrestricted,
     cache,
     typedEndpoint(path as any, handler as any),

backend/api/src/llm-extract-profile.ts

@@ -9,6 +9,8 @@ import {
   GENDERS,
   LANGUAGE_CHOICES,
   MBTI_CHOICES,
+  NEUROTYPE_CHOICES,
+  ORIENTATION_CHOICES,
   POLITICAL_CHOICES,
   PSYCHEDELICS_CHOICES,
   RACE_CHOICES,
@@ -23,7 +25,7 @@ import {debug} from 'common/logger'
 import {ProfileWithoutUser} from 'common/profiles/profile'
 import {SITE_ORDER} from 'common/socials'
 import {removeNullOrUndefinedProps} from 'common/util/object'
-import {parseJsonContentToText} from 'common/util/parse'
+import {parseJsonContentToText, textToJSONContent} from 'common/util/parse'
 import {HOUR_MS, MINUTE_MS, sleep} from 'common/util/time'
 import {createHash} from 'crypto'
 import {promises as fs} from 'fs'
@@ -38,18 +40,26 @@ const CACHE_DIR = join(tmpdir(), 'compass-llm-cache')
 const CACHE_TTL_MS = 24 * HOUR_MS
 const PROCESSING_TTL_MS = 10 * MINUTE_MS
 
+type ExtractSource = 'text' | 'url' | 'voice'
+
 interface ParsedBody {
   content?: string
   url?: string
   locale?: string
+  source?: ExtractSource
 }
 
+// Bump whenever the extraction prompt changes. The cache key is otherwise derived purely from the
+// request, so a prompt fix would keep returning the old answer for the 24h TTL — which looks exactly
+// like the fix not working.
+const PROMPT_VERSION = 2
+
 function getCacheKey(parsedBody: ParsedBody): string {
   if (!USE_CACHE) return ''
   const hash = createHash('sha256')
   // Normalize: sort keys for consistent hashing
   const normalized = JSON.stringify(parsedBody, Object.keys(parsedBody).sort())
-  hash.update(normalized)
+  hash.update(`v${PROMPT_VERSION}:${normalized}`)
   return hash.digest('hex')
 }
 
@@ -79,6 +89,8 @@ async function validateProfileFields(
     'cannabis_intention',
     'psychedelics_pref',
     'cannabis_pref',
+    'orientation',
+    'neurotype',
   ]
   for (const key of toArray) {
     if (result[key] !== undefined) {
@@ -113,6 +125,10 @@ async function validateProfileFields(
     'occupation_title',
     'religious_beliefs',
     'political_details',
+    'gender_details',
+    'orientation_details',
+    'neurotype_details',
+    'accessibility_notes',
   ]
   for (const key of toString) {
     if (result[key] !== undefined) {
@@ -246,7 +262,10 @@ async function setCachedResult(cacheKey: string, result: any): Promise<void> {
     await fs.mkdir(CACHE_DIR, {recursive: true})
     const cacheFile = join(CACHE_DIR, `${cacheKey}.json`)
     await fs.writeFile(cacheFile, JSON.stringify(result), 'utf-8')
-    debug('Cached LLM result', {cacheKey: cacheKey.substring(0, 8), result})
+    debug('Cached LLM result', {
+      cacheKey: cacheKey.substring(0, 8),
+      result: JSON.stringify(result),
+    })
   } catch (error) {
     log('Failed to write cache', {cacheKey, error})
     // Don't throw - caching failure shouldn't break the main flow
@@ -296,11 +315,13 @@ async function processAndCache(
   content?: string | undefined,
   url?: string | undefined,
   locale?: string,
+  source?: ExtractSource,
 ): Promise<void> {
   log('Extracting profile from content', {
     contentLength: content?.length,
     url,
     locale,
+    source,
   })
   try {
     let bio: JSONContent | undefined
@@ -309,7 +330,7 @@ async function processAndCache(
       debug(JSON.stringify(bio, null, 2))
       content = parseJsonContentToText(bio)
     }
-    const profile = await callLLM(content, locale)
+    const profile = await callLLM(content, locale, source)
     if (bio) {
       profile.bio = bio
     }
@@ -422,7 +443,9 @@ async function _callClaude(text: string) {
 export async function callLLM(
   content: string,
   locale?: string,
+  source?: ExtractSource,
 ): Promise<Partial<ProfileWithoutUser>> {
+  const isVoice = source === 'voice'
   const [INTERESTS, CAUSE_AREAS, WORK_AREAS] = await Promise.all([
     getOptions('interests', locale),
     getOptions('causes', locale),
@@ -451,12 +474,19 @@ export async function callLLM(
     cannabis_intention: Object.values(SUBSTANCE_INTENTION_CHOICES),
     psychedelics_pref: Object.values(SUBSTANCE_PREFERENCE_CHOICES),
     cannabis_pref: Object.values(SUBSTANCE_PREFERENCE_CHOICES),
+    orientation: Object.values(ORIENTATION_CHOICES),
+    neurotype: Object.values(NEUROTYPE_CHOICES),
   }
 
   const PROFILE_FIELDS: Partial<Record<keyof ProfileWithoutUser, any>> = {
     // Basic info
     age: 'Number. Age in years (between 18 and 100).',
     gender: `String. One of: ${validChoices.pref_gender?.join(', ')}. If multiple mentioned, use the most likely one. Infer if you have enough evidence`,
+    gender_details:
+      'String. Free-form elaboration on their gender identity, only if they say more than the label itself.',
+    orientation: `Array. Any of: ${validChoices.orientation?.join(', ')}. Only if stated — never infer from the gender of a partner or of who they are looking for.`,
+    orientation_details:
+      'String. Free-form elaboration on their orientation, only if they say more than the label itself.',
     height_in_inches: 'Number. Height converted to inches.',
     city: 'String. Current city of residence (English spelling).',
     country: 'String. Current country of residence (English spelling).',
@@ -498,6 +528,11 @@ export async function callLLM(
     big5_agreeableness: 'Number 0–100. Only if explicitly self-reported, never infer.',
     big5_neuroticism: 'Number 0–100. Only if explicitly self-reported, never infer.',
 
+    // Neurotype is an identity here, not a diagnosis: only ever take the person's own words for it.
+    neurotype: `Array. Any of: ${validChoices.neurotype?.join(', ')}. Only if they identify this way themselves — never infer it from how they describe their personality, focus, energy or social life.`,
+    neurotype_details:
+      'String. Free-form elaboration on their neurotype, only if they say more than the label itself.',
+
     // Beliefs
     religion: `Array. Any of: ${validChoices.religion?.join(', ')}`,
     religious_beliefs:
@@ -511,7 +546,7 @@ export async function callLLM(
       'Number. Minimum preferred age of match (higher than 18, only if mentioned, do NOT infer).',
     pref_age_max:
       'Number. Maximum preferred age of match (lower than 100, only if mentioned, do NOT infer).',
-    pref_gender: `Array. Any of: ${validChoices.pref_gender?.join(', ')}`,
+    pref_gender: `Array. Any of: ${validChoices.pref_gender?.join(', ')}. Only the genders they actually name as sought. If they say gender does not matter to them, or is unimportant to their attraction, OMIT this field — an omitted field already means "no preference", so listing every option instead is both wrong and unreadable.`,
     pref_relation_styles: `Array. Any of: ${validChoices.pref_relation_styles?.join(', ')}`,
     pref_romantic_styles: `Array. Any of: ${validChoices.pref_romantic_styles?.join(', ')}`,
     relationship_status: `Array. Any of: ${validChoices.relationship_status?.join(', ')}`,
@@ -523,6 +558,8 @@ export async function callLLM(
     headline:
       'String. Summary of who they are, in their own voice (first person). Maximum 200 characters total. Cannot be null.',
     keywords: 'Array of 3–6 short tags summarising the person.',
+    accessibility_notes:
+      'String. Practical things that help someone meet them well — access needs, energy levels, sensory preferences, venue preferences. Only if mentioned; never infer a disability, and keep their own framing and wording.',
     links: `Object. Key is any of: ${SITE_ORDER.join(', ')}.`,
 
     // Taxonomies — match existing labels first, only add new if truly no close match exists
@@ -531,7 +568,25 @@ export async function callLLM(
     work: `Array. Use only existing labels, do not add new if no close match. Any of: ${validChoices.work?.join(', ')}`,
   }
 
-  const EXTRACTION_PROMPT = `You are a profile information extraction expert analyzing text from a personal webpage, bio, or similar source.
+  // For text and URL sources the bio is the source material itself, stored verbatim. A speech
+  // transcript makes a poor bio (filler words, false starts, no paragraphs), so for voice we ask
+  // the model to write it instead.
+  if (isVoice) {
+    PROFILE_FIELDS.bio =
+      'String. A first-person bio written from what the person said, in their own voice and their ' +
+      'own language. Keep their wording and personality wherever you can; only clean up filler ' +
+      'words, false starts, repetitions and transcription noise, and organise it into ' +
+      'paragraphs. Separate each paragraph from the next with a BLANK LINE, i.e. two newline ' +
+      'characters ("\\n\\n") — a single newline is not enough. Never add facts, opinions or ' +
+      'flourishes they did not say, and never write about them in the third person. Plain text ' +
+      'only — no markdown.'
+  }
+
+  const EXTRACTION_PROMPT = `You are a profile information extraction expert analyzing ${
+    isVoice
+      ? 'a speech-to-text transcript of someone talking about themselves out loud'
+      : 'text from a personal webpage, bio, or similar source'
+  }.
 
 TASK: Extract structured profile data and return it as a single valid JSON object.
 
@@ -540,12 +595,20 @@ RULES:
 - Omit the key in the output for missing fields
 - For taxonomy fields (interests, causes, work): match existing labels first; only add a new label if truly no existing one is close
 - For big5 scores: only populate if the person explicitly states a test result — never infer from personality description
-- Return valid JSON only — no markdown, no explanation, no extra text
+- Never answer a multi-choice field by selecting every option it offers. An expression of openness or indifference ("gender doesn't matter to me", "I'm open to anything") is not a selection of all values — omit the field, which already means "no preference"
+- Return valid JSON only — no markdown, no explanation, no extra text${
+    isVoice
+      ? `
+- The transcript is spoken language: expect filler words, false starts, self-corrections and speech-recognition errors. Read past them, and when the person corrects themselves keep the corrected version
+- Ignore anything the person says to the recorder rather than about themselves (e.g. "let me start over", "what else should I say")
+- Spoken numbers, places and names may be mis-transcribed; only fill a field when you are confident what was meant`
+      : ''
+  }
 
 SCHEMA (each value describes the expected type and accepted values):
 ${JSON.stringify(PROFILE_FIELDS, null, 2)}
 
-TEXT TO ANALYZE:
+${isVoice ? 'TRANSCRIPT TO ANALYZE' : 'TEXT TO ANALYZE'}:
 `
   const text = EXTRACTION_PROMPT + content
   if (text.length > MAX_CONTEXT_LENGTH) {
@@ -564,6 +627,10 @@ TEXT TO ANALYZE:
   try {
     parsed = typeof outputText === 'string' ? JSON.parse(outputText) : outputText
     parsed = await validateProfileFields(parsed, validChoices)
+    // The bio column holds rich text; the model answers with plain prose.
+    if (typeof parsed.bio === 'string') {
+      parsed.bio = parsed.bio.trim() ? textToJSONContent(parsed.bio) : undefined
+    }
     parsed = removeNullOrUndefinedProps(parsed)
   } catch (parseError) {
     log('Failed to parse LLM response as JSON', {outputText, parseError})
@@ -647,7 +714,7 @@ export async function fetchOnlineProfile(url: string | undefined): Promise<JSONC
 }
 
 export const llmExtractProfileEndpoint: APIHandler<'llm-extract-profile'> = async (parsedBody) => {
-  const {url, locale} = parsedBody
+  const {url, locale, source} = parsedBody
   const content = parsedBody.content
 
   if (content && url) {
@@ -672,7 +739,7 @@ export const llmExtractProfileEndpoint: APIHandler<'llm-extract-profile'> = asyn
   await setProcessing(cacheKey)
 
   // Kick off async processing (don't await)
-  processAndCache(cacheKey, content, url, locale).catch((err) => {
+  processAndCache(cacheKey, content, url, locale, source).catch((err) => {
     log('Unexpected error in async processing', {cacheKey, error: err})
   })
 

backend/api/src/transcribe-audio.ts

@@ -0,0 +1,100 @@
+import {APIErrors, APIHandler} from 'api/helpers/endpoint'
+import {debug} from 'common/logger'
+import {log} from 'shared/monitoring/log'
+
+// Whisper's own hard limit is 25 MB. We cap below that so we reject with a friendly message rather
+// than letting OpenAI 413 on us. Opus runs ~200-400 KB/minute, so this is many minutes of speech.
+const MAX_AUDIO_BYTES = 20 * 1024 * 1024
+
+// `gpt-4o-transcribe` is Whisper's successor and the default in our other voice codebase; set
+// OPENAI_TRANSCRIBE_MODEL=whisper-1 to fall back to classic Whisper.
+const TRANSCRIBE_MODEL = process.env.OPENAI_TRANSCRIBE_MODEL || 'gpt-4o-transcribe'
+
+// Container → file extension. OpenAI infers the codec from the uploaded filename, so this has to be
+// right; `;codecs=...` suffixes that browsers append are stripped first.
+const EXTENSION_BY_MIME: Record<string, string> = {
+  'audio/webm': 'webm',
+  'audio/ogg': 'ogg',
+  'audio/mp4': 'mp4',
+  'audio/m4a': 'm4a',
+  'audio/x-m4a': 'm4a',
+  'audio/mpeg': 'mp3',
+  'audio/mp3': 'mp3',
+  'audio/wav': 'wav',
+  'audio/x-wav': 'wav',
+  'audio/flac': 'flac',
+}
+
+function parseMimeType(mimeType: string): {contentType: string; extension: string} {
+  const base = mimeType.split(';')[0].trim().toLowerCase()
+  const extension = EXTENSION_BY_MIME[base]
+  if (extension) return {contentType: base, extension}
+  // Unknown container: webm/opus is what every Chromium browser records, so it is the safest guess.
+  log('Unsupported audio mime type, falling back to audio/webm', {mimeType})
+  return {contentType: 'audio/webm', extension: 'webm'}
+}
+
+// Whisper takes an ISO-639-1 code; our locales are either 'en' or 'en-US'-shaped.
+function toLanguageCode(locale: string | undefined): string | undefined {
+  const code = locale?.split(/[-_]/)[0]?.toLowerCase()
+  return code && /^[a-z]{2}$/.test(code) ? code : undefined
+}
+
+export const transcribeAudio: APIHandler<'transcribe-audio'> = async (props) => {
+  const {audio, mimeType, locale} = props
+
+  const apiKey = process.env.OPENAI_API_KEY
+  if (!apiKey) {
+    log('OPENAI_API_KEY not configured')
+    throw APIErrors.internalServerError('Voice transcription is not configured')
+  }
+
+  // `audio` is base64: 4 characters encode 3 bytes.
+  const sizeInBytes = Math.floor((audio.length * 3) / 4)
+  if (sizeInBytes > MAX_AUDIO_BYTES) {
+    throw APIErrors.badRequest('Recording is too long. Please record a shorter message.')
+  }
+
+  const {contentType, extension} = parseMimeType(mimeType)
+  const language = toLanguageCode(locale)
+  log('Transcribing audio', {mimeType, contentType, sizeInBytes, model: TRANSCRIBE_MODEL, language})
+
+  const form = new FormData()
+  form.append(
+    'file',
+    new Blob([Buffer.from(audio, 'base64')], {type: contentType}),
+    `voice.${extension}`,
+  )
+  form.append('model', TRANSCRIBE_MODEL)
+  // Punctuated, readable text — the extraction step downstream reads much better prose than a
+  // single unbroken run of words.
+  form.append('response_format', 'text')
+  if (language) form.append('language', language)
+
+  let response: Response
+  try {
+    response = await fetch('https://api.openai.com/v1/audio/transcriptions', {
+      method: 'POST',
+      headers: {Authorization: `Bearer ${apiKey}`},
+      body: form,
+    })
+  } catch (error) {
+    log('OpenAI transcription request failed', {error})
+    throw APIErrors.internalServerError('Failed to transcribe the recording')
+  }
+
+  if (!response.ok) {
+    const errorText = await response.text()
+    log('OpenAI transcription API error', {status: response.status, error: errorText})
+    throw APIErrors.internalServerError('Failed to transcribe the recording')
+  }
+
+  const transcript = (await response.text()).trim()
+  debug({transcript})
+
+  if (!transcript) {
+    throw APIErrors.badRequest('We could not hear any speech in that recording. Please try again.')
+  }
+
+  return {transcript}
+}