diff --git a/apps/light/CMakeLists.txt b/apps/light/CMakeLists.txt index 02d35d69..ba59f3e8 100644 --- a/apps/light/CMakeLists.txt +++ b/apps/light/CMakeLists.txt @@ -229,9 +229,12 @@ if(MONKY_LIGHT_WITH_RTC) target_compile_definitions(monky_light_ui PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX) else() find_library(MONKY_APPKIT_FRAMEWORK AppKit REQUIRED) + # AXIsProcessTrusted: push-to-talk needs Accessibility to see keys. + find_library(MONKY_APPLICATION_SERVICES_FRAMEWORK ApplicationServices REQUIRED) add_library(monky_light_ui STATIC src/ui/tray_macos.mm) set_source_files_properties(src/ui/tray_macos.mm PROPERTIES COMPILE_OPTIONS "-fobjc-arc") - target_link_libraries(monky_light_ui PUBLIC "${MONKY_APPKIT_FRAMEWORK}") + target_link_libraries(monky_light_ui PUBLIC "${MONKY_APPKIT_FRAMEWORK}" + "${MONKY_APPLICATION_SERVICES_FRAMEWORK}") endif() target_include_directories(monky_light_ui PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src") target_link_libraries(monky_light_ui PUBLIC monky_light_json) diff --git a/apps/light/QA.en.md b/apps/light/QA.en.md index 722b4cd2..71c6377c 100644 --- a/apps/light/QA.en.md +++ b/apps/light/QA.en.md @@ -138,3 +138,12 @@ available, so the terminal shows what each click did. | 6.19 | Measure the same phase with and without `--tray` | The tray does not measurably change idle resource use | | | | 6.20 | A channel without `Speak` | The mute row is marked and disabled, with the note that there is no permission to speak | | | | 6.21 | Close the console window during a call (Windows only) | The others see Light leave at once (logout), the icon disappears and no process is left | | | +| 6.22 | Start with `--ptt-key F13` (or `Mouse4`) and turn `Push-to-talk` on | The row is checked and shows the key; nothing is watched outside a call | | | +| 6.23 | In a call, hold and release the key with another application in front | Others only hear you while the key is held, and the start of speech is not cut; the key still reaches the other application | | | +| 6.24 | Hold the key for several seconds | A single `ptt` `active:true` in the terminal, no repetition | | | +| 6.25 | Hold the key and turn `Push-to-talk` off from the menu | Transmits all the time again, with no `command-error` | | | +| 6.26 | Without Accessibility permission (macOS only) | The system asks once and the notice appears in the menu; after allowing it and reopening the menu, the key works | | | +| 6.27 | Speak in voice-activity mode | The icon turns light green while you speak and back to green when you stop | | | +| 6.28 | Open `In the call` while someone speaks | They appear with `(speaking)`; you are not listed | | | +| 6.29 | Choose 0%, 200%, and 100% for a participant | Silent, louder, back to normal; nobody else notices a difference | | | +| 6.30 | Restart the Light and reopen `In the call` | The chosen volume is still checked and applied | | | diff --git a/apps/light/QA.md b/apps/light/QA.md index 75e8a5ea..de1cce40 100644 --- a/apps/light/QA.md +++ b/apps/light/QA.md @@ -139,3 +139,12 @@ valendo, então o terminal serve para conferir o que cada clique fez. | 6.19 | Medir a mesma fase com `--tray` e sem | A bandeja não muda o consumo em repouso de forma perceptível | | | | 6.20 | Canal sem `Falar` | O mute fica marcado e desabilitado, com a nota de que falta permissão para falar | | | | 6.21 | Fechar a janela do console durante uma chamada (só Windows) | Os outros veem o Light sair na hora (logout), o ícone desaparece e não sobra processo | | | +| 6.22 | Iniciar com `--ptt-key F13` (ou `Mouse4`) e ligar `Apertar para falar` | O item fica marcado e mostra a tecla; fora de chamada nada é observado | | | +| 6.23 | Em chamada, segurar e soltar a tecla com outro aplicativo em primeiro plano | Os outros só ouvem enquanto a tecla está segurada, e o início da fala não é cortado; a tecla continua chegando ao outro aplicativo | | | +| 6.24 | Segurar a tecla por vários segundos | Um único `ptt` `active:true` no terminal, sem repetição | | | +| 6.25 | Segurar a tecla e desligar `Apertar para falar` pelo menu | Volta a transmitir sempre, sem `command-error` | | | +| 6.26 | Sem autorização de Acessibilidade (só macOS) | O sistema pede a autorização uma vez e o aviso aparece no menu; após autorizar e reabrir o menu, a tecla funciona | | | +| 6.27 | Falar com o modo de atividade de voz | O ícone fica verde claro enquanto você fala e volta ao verde ao parar | | | +| 6.28 | Abrir `Na chamada` enquanto alguém fala | A pessoa aparece com `(falando)`; você não aparece na lista | | | +| 6.29 | Escolher 0%, 200% e 100% para um participante | Some, fica mais alto, volta ao normal; os outros não percebem diferença | | | +| 6.30 | Reiniciar o Light e reabrir `Na chamada` | O volume escolhido continua marcado e aplicado | | | diff --git a/apps/light/README.en.md b/apps/light/README.en.md index d9307ceb..f90c5a9d 100644 --- a/apps/light/README.en.md +++ b/apps/light/README.en.md @@ -82,6 +82,9 @@ Enter one JSON object per line in the terminal, using the received channel ID: {"command":"devices"} {"command":"set-input","deviceId":"DEVICE-ID"} {"command":"set-output","deviceId":null} +{"command":"set-input-mode","mode":"push_to_talk"} +{"command":"ptt","active":true} +{"command":"set-volume","sessionId":"SESSION-ID","volume":150} {"command":"stats","id":"my-query"} {"command":"leave"} {"command":"reconnect"} @@ -129,6 +132,47 @@ quitting remain available. Denial keeps the microphone muted and emits `microphone-error`; it does not try to bypass system settings. The macOS permission flow still requires execution on a Mac. +### Speaking indicator and push-to-talk + +During a call, the core emits `speaking` with `sessionId` and +`speaking:true|false` whenever someone, including the local user, starts or +stops speaking. As in the full client, this is **not** published to the server: +each client derives who is speaking from the audio. + +- **Remote participants:** the level comes from the RTP header (RFC 6464) the + sender already includes, read every 100 ms. Nothing is decoded or measured + separately. While the user is deafened, no one is shown speaking. +- **Local user:** the level is measured on raw capture, before echo + cancellation and noise suppression. Loud, steady background noise can count + as speech. +- The threshold is -40 dBov (the 0.01 `audioLevel` the full client uses), and + speech ends only after 400 ms of silence so it does not flicker between words. +- Sampling exists only during a call; outside one, no thread wakes up. + +`set-input-mode` selects `voice_activity` (default: always transmitting) or +`push_to_talk` and saves the choice in the profile. In push-to-talk, `ptt` with +`active:true`/`false` opens and closes transmission. The core does not capture +keys: the global hotkey belongs to whatever controls the process; the tray has +no global shortcuts yet. Capture stays open while the key is released, so a press takes +effect immediately. The track sends silence, and the local user is not shown +speaking. Mute, deafen, and server restrictions still apply while the key is +held. Switching modes or leaving the call releases the key. `ptt` outside +push-to-talk mode is rejected. + +### Per-participant volume + +`set-volume` sets, from 0 to 200%, how loud you hear one voice session +(`sessionId` from the participant list), like the full client's slider. It is a +local preference: neither the server nor the other participants know about it. +The choice is saved in the profile, applies at once to the current call and to +later ones, in both P2P and SFU, and emits `participant-volume`. `stats` lists in +`userVolumes` the non-default volumes of the current call's sessions. + +WebRTC applies the gain to the decoded stream before mixing: there is no extra +processing, and 100% costs nothing. Setting 100 again removes the entry. The +profile keeps up to 256 sessions with a changed volume; beyond that, `set-volume` +is rejected until one is set back to 100. + ## Tray interface `--tray` adds the icon to the Windows notification area or the macOS status bar. @@ -153,6 +197,9 @@ rendering between activations. It carries: - the server voice channels, with the current one marked, and `Leave the channel`; - mute and deafen, marked and disabled when the server imposed them or the channel does not allow speaking; +- `Push-to-talk`, which switches the input mode and shows the key; +- `In the call`, with each participant, who is speaking, and the volume, in + presets from 0 to 200%; - `Audio devices`, with input, output, and `System default`; - `Reconnect now` and `Quit Monky Light`. @@ -162,8 +209,9 @@ and the switch of the web interface. Actual switches and cards belong to the panels, which are not part of this milestone. The icon reports the state with the same colours as the full client: grey -disconnected, accent connected without a call, green in a call, and red with a -diagonal bar when muted or deafened. It is drawn at runtime and does **not** +disconnected, accent connected without a call, green in a call, light green +while you speak, and red with a diagonal bar when muted or deafened. Only your +own speech redraws the icon; other people's appears in the menu when it opens. It is drawn at runtime and does **not** reuse the full Monky icon, because one edition must not pass for the other; the Light's own identity belongs to distribution, which is not implemented yet. On macOS the icon is a template system symbol, which follows the menu bar @@ -185,8 +233,31 @@ The language follows the operating system, between Portuguese and English. Choosing the language inside the Light depends on the settings panel, which does not exist yet. -There are still no native panels, global shortcuts, connection screen, or -installer: the tray starts with the same command-line options as the core. +### Push-to-talk shortcut + +The key is chosen with `--ptt-key` and saved in the profile: `A`–`Z`, `0`–`9`, +`F1`–`F24`, `Mouse4`, `Mouse5` (side buttons), `RightControl`, `RightAlt`, +`RightShift`, `Backquote`, `Insert`, or `Pause`. There is no default key, so it +never takes one the user plays with; without a key, the menu row says how to +choose one. Caps Lock and Num Lock are left out because holding them changes +their system state. macOS reports neither `F21`–`F24` nor `Pause`; the menu says so. + +The shortcut is watched **only while it can transmit**: in a call, in +push-to-talk mode. Otherwise nothing is installed and nothing is spent. The key +is never consumed, so it keeps working in the foreground application, and +keyboard auto-repeat becomes a single press. + +- **Windows:** a low-level keyboard hook or, for `Mouse4`/`Mouse5`, a mouse + hook, installed on the interface thread. The mouse hook is called on every + pointer movement, so it exists only when the key is a button. +- **macOS:** an AppKit global event monitor. The system only delivers keys to an + application allowed under *System Settings › Privacy & Security › + Accessibility*; the Light asks once and shows the notice in the menu. Mouse + buttons need no such permission. The macOS shortcut still requires + qualification on a Mac. + +There are still no native panels, connection screen, or installer: the tray +starts with the same command-line options as the core. ## Shared contracts @@ -226,7 +297,7 @@ require explicit recovery, never silent replacement. Directories containing another application's files are rejected. `monky-light-settings.json` stores only user preferences (currently audio -devices) and never participates in identity validation. It is replaced +devices, the input mode, and each participant's volume) and never participates in identity validation. It is replaced atomically under the profile lock. Invalid or unreadable settings emit a `warning`, fall back to defaults, and are replaced by the next save; an interrupted settings write does not require recovery. diff --git a/apps/light/README.md b/apps/light/README.md index 14f83343..0d6ca148 100644 --- a/apps/light/README.md +++ b/apps/light/README.md @@ -82,6 +82,9 @@ Digite um objeto JSON por linha no terminal, usando o ID recebido: {"command":"devices"} {"command":"set-input","deviceId":"ID-DO-DISPOSITIVO"} {"command":"set-output","deviceId":null} +{"command":"set-input-mode","mode":"push_to_talk"} +{"command":"ptt","active":true} +{"command":"set-volume","sessionId":"ID-DA-SESSAO","volume":150} {"command":"stats","id":"minha-consulta"} {"command":"leave"} {"command":"reconnect"} @@ -130,6 +133,47 @@ precisa transmitir. Enquanto a permissão está pendente, é possível receber mutado e gera `microphone-error`; não tenta contornar as configurações do sistema. O fluxo de permissão do macOS ainda requer execução em um Mac. +### Indicador de fala e push-to-talk + +Durante uma chamada, o núcleo emite `speaking` com `sessionId` e +`speaking:true|false` sempre que alguém, inclusive o próprio usuário, começa ou +para de falar. Como no cliente completo, isso **não** é publicado no servidor: +cada cliente deduz quem fala a partir do áudio. + +- **Participantes remotos:** o nível vem do cabeçalho RTP (RFC 6464) que o + remetente já envia, consultado a cada 100 ms. Nada é decodificado ou medido + à parte. Enquanto o usuário está em deafen, ninguém aparece falando. +- **Próprio usuário:** o nível é medido na captura bruta, antes do cancelamento + de eco e da supressão de ruído. Ruído de fundo alto e constante pode contar + como fala. +- O limiar é -40 dBov (o `audioLevel` 0,01 usado pelo cliente completo), e a + fala só termina após 400 ms de silêncio, para não piscar entre palavras. +- A amostragem só existe durante a chamada; fora dela nenhuma thread acorda. + +`set-input-mode` escolhe `voice_activity` (padrão: transmite sempre) ou +`push_to_talk` e salva a escolha no perfil. Em push-to-talk, `ptt` com +`active:true`/`false` abre e fecha a transmissão. O núcleo não captura teclas: +o atalho global é responsabilidade de quem controla o processo; a bandeja +ainda não tem atalhos globais. A captura continua aberta enquanto a tecla está solta, então +pressionar tem efeito imediato. A faixa envia silêncio, e o próprio usuário não +aparece falando. Mute, deafen e restrições do servidor continuam valendo mesmo +com a tecla pressionada. Trocar de modo ou sair da chamada solta a tecla. +`ptt` fora do modo push-to-talk é recusado. + +### Volume por participante + +`set-volume` ajusta de 0 a 200% o volume com que você ouve uma sessão de voz +(`sessionId` da lista de participantes), como o controle deslizante do cliente +completo. É uma preferência local: o servidor e os outros participantes não +sabem dela. A escolha é salva no perfil, vale imediatamente na chamada atual e +nas próximas, inclusive em P2P e SFU, e gera `participant-volume`. `stats` traz +em `userVolumes` os volumes diferentes de 100 das sessões da chamada atual. + +O ganho é aplicado pelo próprio WebRTC ao fluxo decodificado, antes da mixagem: +não há processamento extra, e 100% não custa nada. Voltar a 100 apaga a entrada. +O perfil guarda até 256 sessões com volume alterado; acima disso, `set-volume` +é recusado até que alguma volte a 100. + ## Interface de bandeja `--tray` adiciona o ícone na área de notificação do Windows ou na barra de status @@ -154,6 +198,9 @@ renderizando entre as ativações. Ele traz: - os canais de voz do servidor, com o canal atual marcado, e `Sair do canal`; - mute e deafen, marcados e desabilitados quando a restrição é do servidor ou quando o canal não permite falar; +- `Apertar para falar`, que alterna o modo de entrada e mostra a tecla; +- `Na chamada`, com cada participante, quem está falando e o volume, em + predefinições de 0 a 200%; - `Dispositivos de áudio`, com entrada, saída e `Padrão do sistema`; - `Reconectar agora` e `Sair do Monky Light`. @@ -163,8 +210,9 @@ selecionável e do switch da interface web. Switch e card propriamente ditos val para os painéis, que não fazem parte desta etapa. O ícone reflete o estado com as mesmas cores do cliente completo: cinza -desconectado, accent conectado sem chamada, verde em chamada e vermelho com uma -barra diagonal quando mudo ou silenciado. Ele é desenhado em tempo de execução e +desconectado, accent conectado sem chamada, verde em chamada, verde claro +enquanto você fala e vermelho com uma barra diagonal quando mudo ou silenciado. +Só a sua fala redesenha o ícone; a de outras pessoas aparece no menu ao abri-lo. Ele é desenhado em tempo de execução e **não** reutiliza o ícone do Monky completo, porque uma edição não deve se passar pela outra; a identidade própria do Light faz parte da distribuição, ainda não implementada. No macOS o ícone é um símbolo de sistema em modo *template*, que @@ -185,8 +233,31 @@ carregam o estado atual, para o leitor de tela não anunciar apenas "ícone". O idioma acompanha o do sistema operacional, entre português e inglês. Escolher o idioma dentro do Light depende do painel de configurações, que ainda não existe. -Ainda não há painéis nativos, atalhos globais, tela de conexão nem instalador: a -bandeja é iniciada com as mesmas opções de linha de comando do núcleo. +### Atalho do apertar para falar + +A tecla é escolhida com `--ptt-key` e salva no perfil: `A`–`Z`, `0`–`9`, +`F1`–`F24`, `Mouse4`, `Mouse5` (botões laterais), `RightControl`, `RightAlt`, +`RightShift`, `Backquote`, `Insert` ou `Pause`. Não há tecla padrão, para não +tomar uma tecla que a pessoa usa em jogo; sem tecla, o item do menu diz como +escolher. Caps Lock e Num Lock ficam de fora porque segurá-los altera o estado +deles no sistema. O macOS não informa `F21`–`F24` nem `Pause`; o menu avisa. + +O atalho é observado **só enquanto pode transmitir**: em chamada e no modo +apertar para falar. Fora disso nada é instalado e não há custo. A tecla nunca é +consumida, então continua funcionando no aplicativo em primeiro plano, e a +repetição automática do teclado vira um único aperto. + +- **Windows:** hook de baixo nível do teclado ou, para `Mouse4`/`Mouse5`, do + mouse, instalado na thread da interface. O hook de mouse é chamado a cada + movimento do ponteiro, por isso só existe quando a tecla é um botão. +- **macOS:** monitor global de eventos do AppKit. O sistema só entrega teclas a + um aplicativo autorizado em *Ajustes do Sistema › Privacidade e Segurança › + Acessibilidade*; o Light pede a autorização uma vez e mostra o aviso no menu. + Os botões do mouse não exigem essa autorização. O atalho no macOS ainda + precisa de qualificação em um Mac. + +Ainda não há painéis nativos, tela de conexão nem instalador: a bandeja é +iniciada com as mesmas opções de linha de comando do núcleo. ## Contratos compartilhados @@ -226,7 +297,7 @@ componentes da identidade exigem recuperação explícita, nunca recriação sil Diretórios com arquivos de outros aplicativos são recusados. `monky-light-settings.json` guarda apenas preferências do usuário (hoje, os -dispositivos de áudio) e nunca participa da validação da identidade. É +dispositivos de áudio, o modo de entrada e o volume de cada participante) e nunca participa da validação da identidade. É substituído de forma atômica sob o lock do perfil. Configurações inválidas ou ilegíveis geram `warning`, usam os padrões e são substituídas no próximo salvamento; uma gravação interrompida das configurações não exige recuperação. diff --git a/apps/light/src/headless_application.cpp b/apps/light/src/headless_application.cpp index c2bad990..115ce05f 100644 --- a/apps/light/src/headless_application.cpp +++ b/apps/light/src/headless_application.cpp @@ -8,6 +8,7 @@ #include "profile_identity.hpp" #include "protocol.hpp" #include "protocol_session.hpp" +#include "push_to_talk_key.hpp" #include "server_invite.hpp" #include @@ -41,6 +42,7 @@ struct Options { bool muted = false; bool deafened = false; bool tray = false; + std::optional pushToTalkKey; // From an invitation link given as --server (#821): the key it names and its password. std::optional invitedServerKey; std::optional invitedPassword; @@ -61,7 +63,8 @@ Options parseOptions(const std::vector& arguments) { else if (flag == "--deafened") result.deafened = true; else if (flag == "--tray") result.tray = true; else { - if (flag != "--profile" && flag != "--server" && flag != "--nickname" && flag != "--channel") { + if (flag != "--profile" && flag != "--server" && flag != "--nickname" && flag != "--channel" && + flag != "--ptt-key") { throw std::invalid_argument("Unknown command-line option; use --help"); } if (++index == arguments.size() || arguments[index].empty()) { @@ -71,7 +74,10 @@ Options parseOptions(const std::vector& arguments) { if (flag == "--profile") result.profile = std::filesystem::path(std::u8string(value.begin(), value.end())); else if (flag == "--server") result.server = value; else if (flag == "--nickname") result.nickname = value; - else result.channel = value; + else if (flag == "--ptt-key") { + if (!parsePushToTalkKey(value)) throw std::invalid_argument("Unknown --ptt-key; see --help"); + result.pushToTalkKey = value; + } else result.channel = value; } } if (!result.profile.is_absolute() || result.server.empty() || result.nickname.empty()) { @@ -112,6 +118,7 @@ const char* mediaEventName(media::EventKind kind) { case media::EventKind::stats: return "media-stats"; case media::EventKind::warning: return "media-warning"; case media::EventKind::devices: return "audio-device-selected"; + case media::EventKind::speaking: return "speaking"; } throw std::logic_error("Unknown media event"); } @@ -212,6 +219,14 @@ class Application final { } catch (const std::exception& error) { output({{"event", "warning"}, {"detail", std::string("Ignoring profile settings: ") + error.what()}}); } + // Saved like any other choice, so later starts and the tray keep the key. + if (options_.pushToTalkKey && options_.pushToTalkKey != settings_.pushToTalkKey) { + try { + setPushToTalkKey(options_.pushToTalkKey); + } catch (const std::invalid_argument& error) { + output({{"event", "warning"}, {"detail", error.what()}}); + } + } if (audio_.watchDevices) { stopWatchingDevices_ = audio_.watchDevices([this] { loop_.post([this] { @@ -347,6 +362,7 @@ class Application final { result.muted = session_.policy().suppressMicrophone() || (audio_.requestMicrophoneAccess && !microphoneAuthorized_); result.deafened = session_.policy().suppressPlayback(); + result.transmit = !settings_.pushToTalk || pushToTalkActive_; return result; } @@ -429,6 +445,11 @@ class Application final { const auto bytes = event.stats.dump().size(); loop_.post([this, generation, event = std::move(event)] { if (generation != session_.generation() || generation != activeGeneration_) return; + if (event.kind == media::EventKind::speaking) { + output({{"event", "speaking"}, {"generation", generation}, {"sessionId", event.session_id}, + {"speaking", event.stats.at("speaking")}}); + return; + } output({{"event", mediaEventName(event.kind)}, {"generation", generation}, {"channelId", event.channel_id}, {"sessionId", event.session_id}, {"detail", event.detail}, {"stats", event.stats}}); @@ -451,6 +472,7 @@ class Application final { config.admitted_participants = roster(); config.policy = audioPolicy(); config.devices = devicePreference(); + for (const auto& [session, volume] : settings_.userVolumes) config.volumes.emplace(session, volume / 100.0); config.network_ignore_mask = audio_.networkIgnoreMask; if (auth.iceServers) { for (const auto& value : *auth.iceServers) { @@ -512,6 +534,104 @@ class Application final { if (active_) active_->select_devices(devicePreference()); } + void setInputMode(const Json& value) { + const auto mode = value.at("mode").get(); + if (mode != "voice_activity" && mode != "push_to_talk") { + throw std::invalid_argument("mode must be voice_activity or push_to_talk"); + } + auto settings = settings_; + settings.pushToTalk = mode == "push_to_talk"; + try { + identity_.saveSettings(settings); + } catch (const std::invalid_argument&) { + throw; + } catch (const std::exception& error) { + throw std::invalid_argument(std::string("Could not save input mode: ") + error.what()); + } + settings_ = std::move(settings); + // A key held while switching modes must not keep transmitting later. + pushToTalkActive_ = false; + applyAudioPolicy(); + output({{"event", "input-mode"}, {"mode", mode}}); + } + + // The key itself is the controller's business (a global hotkey in the tray + // app); the core only gates transmission. + void pushToTalk(bool active) { + if (!settings_.pushToTalk) throw std::invalid_argument("Push-to-talk is not the input mode"); + if (pushToTalkActive_ == active) return; + pushToTalkActive_ = active; + applyAudioPolicy(); + } + + // The key is detected by the interface (a global hook in the tray); the core + // only stores it so every interface and later start agree on it. + void setPushToTalkKey(const std::optional& key) { + if (key && !parsePushToTalkKey(*key)) throw std::invalid_argument("Unknown push-to-talk key"); + auto settings = settings_; + settings.pushToTalkKey = key; + try { + identity_.saveSettings(settings); + } catch (const std::invalid_argument&) { + throw; + } catch (const std::exception& error) { + throw std::invalid_argument(std::string("Could not save push-to-talk key: ") + error.what()); + } + settings_ = std::move(settings); + output({{"event", "push-to-talk-key"}, {"key", key ? Json(*key) : Json(nullptr)}}); + } + + // Who is in the current call, with the names the server sent and the volume + // this profile hears each session at. Empty outside a call. + Json members() const { + auto result = Json::array(); + if (!session_.admission() || !session_.authentication()) return result; + const auto& self = session_.authentication()->sessionId; + for (const auto& [session, participant] : session_.admission()->participants) { + std::string name; + if (participant.user && participant.user->is_object()) { + const auto nickname = participant.user->find("nickname"); + if (nickname != participant.user->end() && nickname->is_string()) name = nickname->get(); + } + const auto volume = settings_.userVolumes.find(session); + result.push_back({{"sessionId", session}, {"userId", participant.userId}, {"name", name}, + {"self", session == self}, + {"volume", volume == settings_.userVolumes.end() ? kDefaultUserVolume : volume->second}}); + } + return result; + } + + void publishParticipants() { + if (!closing_) output({{"event", "participants"}, {"members", members()}}); + } + + void applyAudioPolicy() { + if (active_ && activeGeneration_ == session_.generation()) active_->update_policy(audioPolicy()); + } + + // Same scale and key as the full client: 0-200 percent per voice session. + void setVolume(const Json& value) { + const auto session = value.at("sessionId").get(); + const auto& level = value.at("volume"); + if (!level.is_number_integer() || level.get() < 0 || level.get() > kMaxUserVolume) { + throw std::invalid_argument("volume must be an integer from 0 to 200"); + } + const auto volume = level.get(); + auto settings = settings_; + if (volume == kDefaultUserVolume) settings.userVolumes.erase(session); + else settings.userVolumes[session] = volume; + try { + identity_.saveSettings(settings); + } catch (const std::invalid_argument&) { + throw; + } catch (const std::exception& error) { + throw std::invalid_argument(std::string("Could not save participant volume: ") + error.what()); + } + settings_ = std::move(settings); + if (active_) active_->set_volume(session, volume / 100.0); + output({{"event", "participant-volume"}, {"sessionId", session}, {"volume", volume}}); + } + void stopWatchingDevices() { if (!stopWatchingDevices_) return; auto stop = std::move(stopWatchingDevices_); @@ -554,7 +674,10 @@ class Application final { output({{"event", "authenticated"}, {"userId", auth.userId}, {"sessionId", auth.sessionId}, {"voiceMode", session_.voiceMode() == VoiceMode::sfu ? "sfu" : "p2p"}, {"serverName", auth.server.at("name")}, - {"channels", auth.server.at("channels")}}); + {"channels", auth.server.at("channels")}, + // An interface starts from these instead of asking for state. + {"inputMode", settings_.pushToTalk ? "push_to_talk" : "voice_activity"}, + {"pushToTalkKey", settings_.pushToTalkKey ? Json(*settings_.pushToTalkKey) : Json(nullptr)}}); } break; case SessionEventKind::admitted: @@ -564,13 +687,17 @@ class Application final { {"channelId", session_.admission()->channelId}, {"voiceMode", session_.voiceMode() == VoiceMode::sfu ? "sfu" : "p2p"}}); beginVoice(); + publishParticipants(); break; case SessionEventKind::teardown: if (active_ && activeGeneration_ < event.generation) retireVoice(); + // Like the full client, a key held across calls does not carry over. + pushToTalkActive_ = false; output({{"event", "voice-left"}, {"generation", event.generation}}); + publishParticipants(); break; case SessionEventKind::policyChanged: - if (active_ && activeGeneration_ == session_.generation()) active_->update_policy(audioPolicy()); + applyAudioPolicy(); authorizeMicrophone(); if (!closing_) output({{"event", "voice-policy"}, {"policy", event.payload}}); break; @@ -578,6 +705,7 @@ class Application final { case SessionEventKind::peerJoined: case SessionEventKind::peerLeft: if (active_ && activeGeneration_ == event.generation) active_->update_roster(roster()); + publishParticipants(); break; case SessionEventKind::rtcSignal: if (active_ && activeGeneration_ == event.generation) active_->receive_signal(event.payload); @@ -716,10 +844,23 @@ class Application final { {"voiceMode", session_.voiceMode() == VoiceMode::sfu ? "sfu" : "p2p"}, {"mediaActive", active_ != nullptr}, {"retiringMedia", retired_.size()}, {"policy", session_.policy().describe()}, + {"inputMode", settings_.pushToTalk ? "push_to_talk" : "voice_activity"}, + {"pushToTalkActive", pushToTalkActive_}, {"audioDevice", audio_.diagnostics ? audio_.diagnostics() : Json(nullptr)}}; result["microphoneAccessPending"] = microphoneRequestPending_; result["channelId"] = session_.admission() ? Json(session_.admission()->channelId) : Json(nullptr); result["participants"] = session_.admission() ? session_.admission()->participants.size() : 0; + // Only the current call's sessions: the profile may remember many more. + auto volumes = Json::object(); + if (session_.admission()) { + for (const auto& [session, participant] : session_.admission()->participants) { + const auto found = settings_.userVolumes.find(session); + if (found != settings_.userVolumes.end()) volumes[session] = found->second; + } + } + result["userVolumes"] = std::move(volumes); + result["members"] = members(); + result["pushToTalkKey"] = settings_.pushToTalkKey ? Json(*settings_.pushToTalkKey) : Json(nullptr); result["channels"] = session_.authentication() ? session_.authentication()->server.at("channels") : Json(nullptr); return result; } @@ -752,6 +893,14 @@ class Application final { if (name.starts_with("fixture-") && audio_.fixtureCommand) audio_.fixtureCommand(value); else if (name == "set-input") selectDevice(true, value); else if (name == "set-output") selectDevice(false, value); + else if (name == "set-input-mode") setInputMode(value); + else if (name == "ptt") pushToTalk(value.at("active").get()); + else if (name == "set-volume") setVolume(value); + else if (name == "set-ptt-key") { + const auto& key = value.at("key"); + if (!key.is_null() && !key.is_string()) throw std::invalid_argument("key must be a key name or null"); + setPushToTalkKey(key.is_null() ? std::nullopt : std::optional(key.get())); + } else if (name == "join") session_.join(value.at("channelId").get(), Clock::now()); else if (name == "leave") session_.leave(); else if (name == "mute") session_.setMuted(value.at("enabled").get()); @@ -759,7 +908,7 @@ class Application final { else if (name == "reconnect") restartConnection(); else if (name != "quit") { throw std::invalid_argument("Unknown command; use join, leave, mute, deafen, devices, set-input, " - "set-output, stats, channels, reconnect or quit"); + "set-output, set-input-mode, ptt, set-ptt-key, set-volume, stats, channels, reconnect or quit"); } output({{"event", "command-accepted"}, {"id", id}, {"command", name}}); if (name == "quit") quit(); @@ -799,6 +948,7 @@ class Application final { const bool outputIsConsole_ = standardOutputIsConsole(); bool microphoneAuthorized_ = false; bool microphoneRequestPending_ = false; + bool pushToTalkActive_ = false; CancelMicrophoneAccess cancelMicrophoneAccess_; std::function observer_; int exitCode_ = 0; @@ -818,7 +968,9 @@ int runHeadless(std::vector arguments, HeadlessConfiguration audio) if (arguments == std::vector{"--help"}) { std::cout << "Monky Light native voice core (development)\n" " --profile --server --nickname \n" - " [--channel ] [--muted] [--deafened] [--tray]\n" + " [--channel ] [--muted] [--deafened] [--tray] [--ptt-key ]\n" + "--ptt-key: A-Z, 0-9, F1-F24, Mouse4, Mouse5, RightControl, RightAlt, RightShift,\n" + " Backquote, Insert or Pause; saved in the profile for the tray's push-to-talk.\n" "The profile parent must exist. Password: MONKY_LIGHT_PASSWORD environment variable, else the invitation's.\n" "Control: one JSON object per line on stdin; events are JSON lines on stdout.\n" "{\"command\":\"join\",\"channelId\":\"...\"}\n" @@ -826,6 +978,10 @@ int runHeadless(std::vector arguments, HeadlessConfiguration audio) "{\"command\":\"deafen\",\"enabled\":false}\n" "{\"command\":\"devices\"} / {\"command\":\"set-input\",\"deviceId\":\"...\"|null}\n" "{\"command\":\"set-output\",\"deviceId\":\"...\"|null}\n" + "{\"command\":\"set-input-mode\",\"mode\":\"voice_activity\"|\"push_to_talk\"}\n" + "{\"command\":\"ptt\",\"active\":true|false}\n" + "{\"command\":\"set-ptt-key\",\"key\":\"F13\"|null}\n" + "{\"command\":\"set-volume\",\"sessionId\":\"...\",\"volume\":0-200}\n" "{\"command\":\"stats\"} / {\"command\":\"channels\"}\n" "{\"command\":\"leave\"} / {\"command\":\"reconnect\"} / {\"command\":\"quit\"}\n" "EOF and Ctrl+C release the connection and media resources.\n" diff --git a/apps/light/src/media/policy_audio_device.hpp b/apps/light/src/media/policy_audio_device.hpp index 8b5de5ca..eb416e16 100644 --- a/apps/light/src/media/policy_audio_device.hpp +++ b/apps/light/src/media/policy_audio_device.hpp @@ -1,6 +1,7 @@ #pragma once #include "audio_devices.hpp" +#include "speech_activity.hpp" #include #include @@ -54,6 +55,18 @@ class PolicyAudioDevice : public webrtc::AudioDeviceModule { } const AudioDeviceSelection &input_selection() const { return input_; } const AudioDeviceSelection &output_selection() const { return output_; } + // Safe from any thread: loudest raw capture level since the last read. + std::uint8_t TakeInputLevel() { return meter_.Take(); } + + int32_t RegisterAudioCallback(webrtc::AudioTransport *callback) override { + if (!callback) { + const auto result = native_->RegisterAudioCallback(nullptr); + meter_.SetTarget(nullptr); + return result; + } + meter_.SetTarget(callback); + return native_->RegisterAudioCallback(&meter_); + } int32_t InitRecording() override { return capture_allowed_ ? native_->InitRecording() : 0; @@ -91,8 +104,6 @@ class PolicyAudioDevice : public webrtc::AudioDeviceModule { #define MONKY_ADM_CONST(result, name, parameters, arguments) \ result name parameters const override { return native_->name arguments; } MONKY_ADM_CONST(int32_t, ActiveAudioLayer, (AudioLayer * layer), (layer)) - MONKY_ADM_FORWARD(int32_t, RegisterAudioCallback, - (webrtc::AudioTransport * callback), (callback)) MONKY_ADM_FORWARD(int32_t, Init, (), ()) MONKY_ADM_FORWARD(int32_t, Terminate, (), ()) MONKY_ADM_CONST(bool, Initialized, (), ()) @@ -232,6 +243,7 @@ class PolicyAudioDevice : public webrtc::AudioDeviceModule { } webrtc::scoped_refptr native_; + InputLevelMeter meter_; AudioDeviceSelection input_; AudioDeviceSelection output_; // The first selection always applies, even for the default device. diff --git a/apps/light/src/media/speech_activity.hpp b/apps/light/src/media/speech_activity.hpp new file mode 100644 index 00000000..769166c1 --- /dev/null +++ b/apps/light/src/media/speech_activity.hpp @@ -0,0 +1,137 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace monky::light::media { + +// Levels use the RFC 6464 scale carried by RTP: 0 is full scale, 127 silence, +// in -dBov. Remote levels come from that header; local ones are measured here. +constexpr std::uint8_t kSilentLevel = 127; +// -40 dBov is the 0.01 linear audioLevel the Chromium client treats as voice. +constexpr std::uint8_t kSpeechLevel = 40; +// Consecutive quiet samples before speech ends, so word gaps do not flicker. +constexpr unsigned kSpeechHoldSamples = 4; + +inline std::uint8_t LevelOf(const std::int16_t *samples, std::size_t count) { + if (count == 0) + return kSilentLevel; + double energy = 0; + for (std::size_t index = 0; index < count; ++index) + energy += static_cast(samples[index]) * samples[index]; + const double rms = std::sqrt(energy / static_cast(count)) / 32768.0; + if (rms <= 0) + return kSilentLevel; + return static_cast( + std::clamp(-20.0 * std::log10(rms), 0.0, double{kSilentLevel})); +} + +// Hysteresis over periodic samples. Missing samples (no packets, stopped +// capture) count as silence. Owned by a single thread. +class SpeechDetector { +public: + // Returns the new state only when it changes. + std::optional Sample(std::uint8_t level) { + if (level <= kSpeechLevel) { + quiet_ = 0; + return Set(true); + } + if (++quiet_ < kSpeechHoldSamples) + return std::nullopt; + return Set(false); + } + // Ends speech immediately, e.g. when transmission is gated. + std::optional Silence() { + quiet_ = kSpeechHoldSamples; + return Set(false); + } + bool speaking() const { return speaking_; } + +private: + std::optional Set(bool value) { + if (speaking_ == value) + return std::nullopt; + speaking_ = value; + return value; + } + bool speaking_ = false; + unsigned quiet_ = 0; +}; + +// Sits between the native ADM and WebRTC's audio transport and keeps the +// loudest capture level since the last read. Measuring costs one pass over +// each 10 ms block on the capture thread; nothing is copied or allocated. +// The level is raw device input, before echo cancellation and noise +// suppression, so steady loud background noise can count as speech. +class InputLevelMeter final : public webrtc::AudioTransport { +public: + void SetTarget(webrtc::AudioTransport *target) { target_.store(target); } + // Loudest level since the previous call; silence when nothing was captured. + std::uint8_t Take() { return loudest_.exchange(kSilentLevel); } + + int32_t RecordedDataIsAvailable(const void *samples, size_t count, + size_t bytes_per_sample, size_t channels, + uint32_t rate, uint32_t delay, + int32_t drift, uint32_t mic_level, + bool key_pressed, + uint32_t &new_mic_level) override { + return RecordedDataIsAvailable(samples, count, bytes_per_sample, channels, + rate, delay, drift, mic_level, key_pressed, + new_mic_level, std::nullopt); + } + int32_t RecordedDataIsAvailable( + const void *samples, size_t count, size_t bytes_per_sample, + size_t channels, uint32_t rate, uint32_t delay, int32_t drift, + uint32_t mic_level, bool key_pressed, uint32_t &new_mic_level, + std::optional capture_time_ns) override { + if (bytes_per_sample == channels * sizeof(std::int16_t)) + Record(LevelOf(static_cast(samples), + count * channels)); + auto *target = target_.load(); + if (!target) { + new_mic_level = mic_level; + return 0; + } + return target->RecordedDataIsAvailable( + samples, count, bytes_per_sample, channels, rate, delay, drift, + mic_level, key_pressed, new_mic_level, capture_time_ns); + } + int32_t NeedMorePlayData(size_t count, size_t bytes_per_sample, + size_t channels, uint32_t rate, void *samples, + size_t &samples_out, int64_t *elapsed_ms, + int64_t *ntp_ms) override { + auto *target = target_.load(); + if (!target) { + samples_out = 0; + return 0; + } + return target->NeedMorePlayData(count, bytes_per_sample, channels, rate, + samples, samples_out, elapsed_ms, ntp_ms); + } + void PullRenderData(int bits_per_sample, int rate, size_t channels, + size_t frames, void *data, int64_t *elapsed_ms, + int64_t *ntp_ms) override { + if (auto *target = target_.load()) + target->PullRenderData(bits_per_sample, rate, channels, frames, data, + elapsed_ms, ntp_ms); + } + +private: + void Record(std::uint8_t level) { + auto current = loudest_.load(); + while (level < current && !loudest_.compare_exchange_weak(current, level)) { + } + } + // WebRTC registers with both directions stopped, but test devices do not + // enforce that; an atomic keeps a late callback from reading a torn value. + std::atomic target_{nullptr}; + std::atomic loudest_{kSilentLevel}; +}; + +} // namespace monky::light::media diff --git a/apps/light/src/media/voice_engine.cpp b/apps/light/src/media/voice_engine.cpp index 019ee356..015662c8 100644 --- a/apps/light/src/media/voice_engine.cpp +++ b/apps/light/src/media/voice_engine.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #if defined(WEBRTC_WIN) #include #endif @@ -312,6 +313,15 @@ std::vector Sections(const std::string &sdp) { } } // namespace +// A remote audio source scales its decoded stream before the mixer; nothing +// is added to the audio path and the default gain costs nothing. +void SetGain(webrtc::MediaStreamTrackInterface *track, double gain) { + if (!track || track->kind() != webrtc::MediaStreamTrackInterface::kAudioKind) + return; + if (auto *source = static_cast(track)->GetSource()) + source->SetVolume(gain); +} + struct VoiceEngine::Impl : std::enable_shared_from_this { struct Peer; struct SfuListener; @@ -355,6 +365,7 @@ struct VoiceEngine::Impl : std::enable_shared_from_this { std::set failed_producers; std::map sfu_transport_states; Json applied_devices; + std::map speech; bool initialized = false; // The server refused the microphone: the call only receives, with capture // stopped, until the next policy change tries again. @@ -540,6 +551,14 @@ struct VoiceEngine::Impl : std::enable_shared_from_this { void Consume(const std::string &id); void CloseConsumer(const std::string &id); void Stats(); + void ScheduleSpeech(); + void SampleSpeech(); + void ReportSpeech(const std::string &session, std::optional change); + double Volume(const std::string &session) const { + const auto found = config.volumes.find(session); + return found == config.volumes.end() ? 1.0 : found->second; + } + void ApplyVolume(const std::string &session); }; struct VoiceEngine::Impl::Peer : webrtc::PeerConnectionObserver, @@ -875,6 +894,7 @@ void VoiceEngine::Impl::EnsureMicrophone() { microphone = factory->CreateAudioTrack("monky-microphone", source.get()); if (!microphone) throw std::runtime_error("Could not create microphone track"); + microphone->set_enabled(config.policy.transmit); for (auto &[id, peer] : peers) if (!peer->mic_transceiver->sender()->SetTrack(microphone.get())) throw std::runtime_error("Could not attach microphone to peer " + id); @@ -938,6 +958,12 @@ void VoiceEngine::Impl::ApplyPolicy() { StopMicrophone(true); else EnsureMicrophone(); + if (microphone) + microphone->set_enabled(config.policy.transmit); + // Report gated speech now instead of after the quiet hold. + if (muted || !config.policy.transmit) + ReportSpeech(config.self_session_id, + speech[config.self_session_id].Silence()); for (auto &[id, peer] : peers) ConfigureTracks(*peer); for (auto &[id, consumer] : consumers) { @@ -1042,6 +1068,7 @@ std::string VoiceEngine::Impl::Cleanup() { } environment.reset(); applied_devices = nullptr; + speech.clear(); initialized = false; publish_refused = false; sfu_recovery_pending = false; @@ -1094,6 +1121,7 @@ void VoiceEngine::Impl::Join(JoinConfig value, } token->CheckActive(); initialized = true; + ScheduleSpeech(); Emit(EventKind::initialized, peers.empty() && consumers.empty() ? "Voice initialized; no remote microphone connected" @@ -1264,8 +1292,11 @@ void VoiceEngine::Impl::ConfigureTracks(Peer &peer) { peer.participant.user_id != config.self_user_id && !peer.participant.suppressed && !token->deafened.load(); - if (auto track = transceiver->receiver()->track()) + if (auto track = transceiver->receiver()->track()) { track->set_enabled(listen); + if (listen) + SetGain(track.get(), Volume(peer.participant.session_id)); + } if (transceiver->media_type() == webrtc::MediaType::VIDEO) { if (!transceiver->stopped()) Check(transceiver->StopStandard(), "Reject video transceiver"); @@ -1704,6 +1735,7 @@ void VoiceEngine::Impl::Consume(const std::string &id) { // The transport keeps a pointer to it until it is closed: listed at once, // whatever happens next, it is closed with the call. auto &owned = consumers.emplace(id, std::move(consumer)).first->second; + SetGain(owned->GetTrack(), Volume(session)); if (token->deafened.load()) { owned->Pause(); Notify(msg::SFU_CONSUMER_SET_PAUSED, {{"channelId", config.channel_id}, @@ -1784,6 +1816,77 @@ void VoiceEngine::Impl::Stats() { } } +// Polling at a human-perceptible rate is cheaper than per-frame sinks and keeps +// all speech state on media control. Remote levels are the RFC 6464 values +// the sender put in each RTP header, so nothing is decoded or measured here. +constexpr auto kSpeechSampleInterval = std::chrono::milliseconds(100); +// Opus DTX still sends a packet every 400 ms; older sources mean no stream. +constexpr std::int64_t kSpeechSourceAgeMs = 500; + +void VoiceEngine::Impl::ScheduleSpeech() { + auto weak = weak_from_this(); + PostAfter(kSpeechSampleInterval, [weak, call = token] { + auto self = weak.lock(); + if (!self || !self->Current(call)) + return; + self->SampleSpeech(); + self->ScheduleSpeech(); + }); +} + +void VoiceEngine::Impl::SampleSpeech() { + std::map levels; + const bool deafened = token->deafened.load(); + // Always drain the meter so audio captured while gated is never reported. + const auto local = adm->TakeInputLevel(); + const bool transmitting = microphone && config.policy.transmit && + !token->muted.load() && !deafened; + levels[config.self_session_id] = transmitting ? local : kSilentLevel; + const auto now = webrtc::TimeMillis(); + const auto remote = [&](const std::string &session, + webrtc::RtpReceiverInterface *receiver) { + auto &level = levels.try_emplace(session, kSilentLevel).first->second; + // Deafened users hear no one; SFU consumers are paused anyway. + if (!receiver || deafened) + return; + for (const auto &source : receiver->GetSources()) + if (source.audio_level() && + now - source.timestamp().ms() <= kSpeechSourceAgeMs) + level = std::min(level, *source.audio_level()); + }; + for (const auto &[id, peer] : peers) + if (peer->mic_transceiver && + peer->participant.user_id != config.self_user_id) + remote(id, peer->mic_transceiver->receiver().get()); + for (const auto &[id, consumer] : consumers) + if (const auto found = known_producers.find(id); + found != known_producers.end()) + remote(Text(found->second, "producerSessionId"), + consumer->GetRtpReceiver()); + // Whoever left the call stops speaking at once. + for (auto entry = speech.begin(); entry != speech.end();) { + if (roster.contains(entry->first)) { + ++entry; + continue; + } + ReportSpeech(entry->first, entry->second.Silence()); + entry = speech.erase(entry); + } + for (const auto &[session, level] : levels) + if (roster.contains(session)) + ReportSpeech(session, speech[session].Sample(level)); + for (auto &[session, detector] : speech) + if (!levels.contains(session)) + ReportSpeech(session, detector.Sample(kSilentLevel)); +} + +void VoiceEngine::Impl::ReportSpeech(const std::string &session, + std::optional change) { + if (change) + Emit(EventKind::speaking, *change ? "Speaking" : "Silent", session, + {{"speaking", *change}}); +} + struct VoiceEngine::Owner { std::shared_ptr impl; std::thread thread; @@ -1970,6 +2073,32 @@ void VoiceEngine::update_policy(AudioPolicy policy) { } } +void VoiceEngine::Impl::ApplyVolume(const std::string &session) { + if (const auto peer = peers.find(session); + peer != peers.end() && peer->second->mic_transceiver) + SetGain(peer->second->mic_transceiver->receiver()->track().get(), + Volume(session)); + for (const auto &[id, consumer] : consumers) + if (const auto found = known_producers.find(id); + found != known_producers.end() && + Text(found->second, "producerSessionId") == session) + SetGain(consumer->GetTrack(), Volume(session)); +} + +void VoiceEngine::set_volume(std::string session_id, double gain) { + if (!(gain >= 0.0 && gain <= 2.0)) + throw std::invalid_argument("Participant gain must be between 0 and 2"); + auto self = owner_->impl; + if (auto call = self->Ingress()) + self->ForCall(call, [session = std::move(session_id), gain](Impl &engine) { + if (gain == 1.0) + engine.config.volumes.erase(session); + else + engine.config.volumes[session] = gain; + engine.ApplyVolume(session); + }); +} + void VoiceEngine::select_devices(AudioDevicePreference devices) { auto self = owner_->impl; if (auto call = self->Ingress()) diff --git a/apps/light/src/media/voice_engine.hpp b/apps/light/src/media/voice_engine.hpp index 7874955b..5fb4058a 100644 --- a/apps/light/src/media/voice_engine.hpp +++ b/apps/light/src/media/voice_engine.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,9 @@ struct AudioPolicy { // These are EFFECTIVE values, including server restrictions. bool muted = false; bool deafened = false; + // Push-to-talk gate. The device keeps capturing so a press takes effect + // at once; the track sends silence and local speech is not reported. + bool transmit = true; bool echo_cancellation = true; bool automatic_gain = true; NoiseSuppression noise_suppression = NoiseSuppression::moderate; @@ -54,6 +58,9 @@ struct JoinConfig { std::vector ice_servers; AudioPolicy policy; AudioDevicePreference devices; + // Playback gain per remote session (1.0 when absent), applied to the + // decoded stream before mixing, as the full client's per-user slider. + std::map volumes{}; std::optional network_ignore_mask; std::chrono::milliseconds operation_timeout{15000}; }; @@ -80,7 +87,11 @@ enum class EventKind { stats, warning, // `stats` carries the effective input/output device selection. - devices + devices, + // A participant (self included) started or stopped speaking; `stats` holds + // {"speaking": bool}. Sampled only during a call, never published to the + // server: receivers derive it from audio, as the Chromium client does. + speaking }; struct Event { EventKind kind; @@ -127,6 +138,8 @@ class VoiceEngine final { // Persists for later generations of this call. Missing devices fall back to // the system default and are retried when the preference is applied again. void select_devices(AudioDevicePreference devices); + // Gain 0.0-2.0; applies at once and to that session's later streams. + void set_volume(std::string session_id, double gain); void poll_stats(); private: diff --git a/apps/light/src/profile_identity.cpp b/apps/light/src/profile_identity.cpp index 95f6df91..46cd61ba 100644 --- a/apps/light/src/profile_identity.cpp +++ b/apps/light/src/profile_identity.cpp @@ -2,6 +2,7 @@ #include "identity_key.hpp" #include "platform/identity_store.hpp" +#include "push_to_talk_key.hpp" #include @@ -46,7 +47,10 @@ constexpr std::size_t kMaxMetadataBytes = 4096; constexpr char kSettingsFile[] = "monky-light-settings.json"; constexpr char kSettingsPendingFile[] = "monky-light-settings.json.pending"; constexpr char kSettingsFormat[] = "monky-light-settings"; -constexpr std::size_t kMaxSettingsBytes = 16 * 1024; +constexpr std::size_t kMaxSettingsBytes = 64 * 1024; +// Same names as the full client's settings. +constexpr char kVoiceActivity[] = "voice_activity"; +constexpr char kPushToTalk[] = "push_to_talk"; constexpr std::size_t kMaxDeviceIdBytes = 1024; // server-key-auth (#821): the identity key pinned for each server this profile signed in to. constexpr char kServerKeysFile[] = "monky-light-servers.json"; @@ -533,12 +537,43 @@ std::optional settingsDevice(const Json& data, const char* field) { return found->get(); } +std::optional settingsKey(const Json& data) { + const auto found = data.find("pushToTalkKey"); + if (found == data.end() || found->is_null()) return std::nullopt; + if (!found->is_string() || !parsePushToTalkKey(found->get())) { + throw std::runtime_error("Profile settings contain an unknown push-to-talk key"); + } + return found->get(); +} + +bool validUserVolume(const std::string& session, int volume) { + return !session.empty() && session.size() <= kMaxVolumeSessionBytes && volume >= 0 && + volume <= kMaxUserVolume && volume != kDefaultUserVolume; +} + +std::map settingsVolumes(const Json& data) { + std::map result; + const auto found = data.find("userVolumes"); + if (found == data.end() || found->is_null()) return result; + if (!found->is_object() || found->size() > kMaxUserVolumes) { + throw std::runtime_error("Profile settings contain invalid participant volumes"); + } + for (const auto& [session, volume] : found->items()) { + if (!volume.is_number_integer() || !validUserVolume(session, volume.get())) { + throw std::runtime_error("Profile settings contain invalid participant volumes"); + } + result.emplace(session, volume.get()); + } + return result; +} + ProfileSettings parseSettings(const std::string& contents) { Json data; try { + // Only the participant-volume map nests, one level deep. data = Json::parse(contents, [](int depth, Json::parse_event_t event, Json&) { if (event == Json::parse_event_t::array_start || - (event == Json::parse_event_t::object_start && depth != 0)) { + (event == Json::parse_event_t::object_start && depth > 1)) { throw std::runtime_error("Profile settings must be a flat JSON object"); } return true; @@ -550,7 +585,18 @@ ProfileSettings parseSettings(const std::string& contents) { !data.contains("version") || !data["version"].is_number_integer() || data["version"] != 1) { throw std::runtime_error("Profile settings have an unsupported format or version"); } - return {settingsDevice(data, "inputDeviceId"), settingsDevice(data, "outputDeviceId")}; + for (const auto& [key, value] : data.items()) { + if (value.is_object() && key != "userVolumes") { + throw std::runtime_error("Profile settings must be a flat JSON object"); + } + } + // Absent in profiles saved before push-to-talk existed. + const auto mode = data.value("inputMode", Json(kVoiceActivity)); + if (mode != kVoiceActivity && mode != kPushToTalk) { + throw std::runtime_error("Profile settings contain an invalid input mode"); + } + return {settingsDevice(data, "inputDeviceId"), settingsDevice(data, "outputDeviceId"), mode == kPushToTalk, + settingsVolumes(data), settingsKey(data)}; } std::string encodeSettings(const ProfileSettings& settings) { @@ -559,6 +605,17 @@ std::string encodeSettings(const ProfileSettings& settings) { throw std::invalid_argument("Audio device ID must be a nonempty string of at most 1024 bytes"); } } + if (settings.pushToTalkKey && !parsePushToTalkKey(*settings.pushToTalkKey)) { + throw std::invalid_argument("Unknown push-to-talk key"); + } + if (settings.userVolumes.size() > kMaxUserVolumes) { + throw std::invalid_argument("Too many participant volumes are stored; reset some to 100"); + } + for (const auto& [session, volume] : settings.userVolumes) { + if (!validUserVolume(session, volume)) { + throw std::invalid_argument("Participant volume must be 0-200 for a session ID of at most 256 bytes"); + } + } const auto optional = [](const std::optional& value) { return value ? Json(*value) : Json(nullptr); }; @@ -566,9 +623,12 @@ std::string encodeSettings(const ProfileSettings& settings) { try { result = Json{{"format", kSettingsFormat}, {"version", 1}, {"inputDeviceId", optional(settings.inputDeviceId)}, - {"outputDeviceId", optional(settings.outputDeviceId)}}.dump(2) + "\n"; + {"outputDeviceId", optional(settings.outputDeviceId)}, + {"inputMode", settings.pushToTalk ? kPushToTalk : kVoiceActivity}, + {"userVolumes", Json(settings.userVolumes)}, + {"pushToTalkKey", optional(settings.pushToTalkKey)}}.dump(2) + "\n"; } catch (const Json::exception&) { - throw std::invalid_argument("Audio device ID must be valid UTF-8"); + throw std::invalid_argument("Audio device and session IDs must be valid UTF-8"); } if (result.size() > kMaxSettingsBytes) throw std::invalid_argument("Profile settings exceed their size limit"); return result; diff --git a/apps/light/src/profile_identity.hpp b/apps/light/src/profile_identity.hpp index 43c5ef33..4301dbf7 100644 --- a/apps/light/src/profile_identity.hpp +++ b/apps/light/src/profile_identity.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -16,8 +17,20 @@ namespace monky::light { struct ProfileSettings { std::optional inputDeviceId; // nullopt: system default std::optional outputDeviceId; // nullopt: system default + bool pushToTalk = false; // false: voice activity + // Playback volume per voice session, 0-200 percent like the full client. + // 100 is the default and is never stored. + std::map userVolumes{}; + // A name accepted by parsePushToTalkKey; nullopt until the user picks one. + std::optional pushToTalkKey{}; }; +constexpr int kDefaultUserVolume = 100; +constexpr int kMaxUserVolume = 200; +// Bounds the settings file; setting a volume back to 100 frees its entry. +constexpr std::size_t kMaxUserVolumes = 256; +constexpr std::size_t kMaxVolumeSessionBytes = 256; + // Owns the signing key and exclusive profile lock for its entire lifetime. // Requires an absolute path and an existing parent; creates only the profile leaf. // Existing directories must contain only this component's identity and settings files. diff --git a/apps/light/src/push_to_talk_key.hpp b/apps/light/src/push_to_talk_key.hpp new file mode 100644 index 00000000..dc3b526c --- /dev/null +++ b/apps/light/src/push_to_talk_key.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace monky::light { + +// The push-to-talk key, by a platform-neutral name saved in the profile. Each +// platform maps the name onto its own key codes. Keys whose state the system +// keeps (Caps Lock, Num Lock) are left out: holding them would toggle it. +struct PushToTalkKey { + enum class Kind { keyboard, mouse }; + Kind kind = Kind::keyboard; + std::string name; + // Windows virtual-key code (VK_*), or the XBUTTON number for the mouse. + std::uint16_t windows = 0; + // macOS virtual key code (kVK_*), or the NSEvent buttonNumber for the mouse. + // Absent when macOS cannot report that key. + std::optional mac; +}; + +namespace push_to_talk_detail { + +struct Named { + std::string_view name; + std::uint16_t windows; + std::optional mac; +}; + +// Windows VK_* and macOS kVK_* (Carbon Events.h) values, written out so this +// header stays platform-free. F21-F24 and Pause have no macOS key code. +inline constexpr std::array kKeys{{ + {"A", 0x41, 0x00}, {"B", 0x42, 0x0B}, {"C", 0x43, 0x08}, {"D", 0x44, 0x02}, + {"E", 0x45, 0x0E}, {"F", 0x46, 0x03}, {"G", 0x47, 0x05}, {"H", 0x48, 0x04}, + {"I", 0x49, 0x22}, {"J", 0x4A, 0x26}, {"K", 0x4B, 0x28}, {"L", 0x4C, 0x25}, + {"M", 0x4D, 0x2E}, {"N", 0x4E, 0x2D}, {"O", 0x4F, 0x1F}, {"P", 0x50, 0x23}, + {"Q", 0x51, 0x0C}, {"R", 0x52, 0x0F}, {"S", 0x53, 0x01}, {"T", 0x54, 0x11}, + {"U", 0x55, 0x20}, {"V", 0x56, 0x09}, {"W", 0x57, 0x0D}, {"X", 0x58, 0x07}, + {"Y", 0x59, 0x10}, {"Z", 0x5A, 0x06}, + {"0", 0x30, 0x1D}, {"1", 0x31, 0x12}, {"2", 0x32, 0x13}, {"3", 0x33, 0x14}, + {"4", 0x34, 0x15}, {"5", 0x35, 0x17}, {"6", 0x36, 0x16}, {"7", 0x37, 0x1A}, + {"8", 0x38, 0x1C}, {"9", 0x39, 0x19}, + {"F1", 0x70, 0x7A}, {"F2", 0x71, 0x78}, {"F3", 0x72, 0x63}, {"F4", 0x73, 0x76}, + {"F5", 0x74, 0x60}, {"F6", 0x75, 0x61}, {"F7", 0x76, 0x62}, {"F8", 0x77, 0x64}, + {"F9", 0x78, 0x65}, {"F10", 0x79, 0x6D}, {"F11", 0x7A, 0x67}, {"F12", 0x7B, 0x6F}, + {"F13", 0x7C, 0x69}, {"F14", 0x7D, 0x6B}, {"F15", 0x7E, 0x71}, {"F16", 0x7F, 0x6A}, + {"F17", 0x80, 0x40}, {"F18", 0x81, 0x4F}, {"F19", 0x82, 0x50}, {"F20", 0x83, 0x5A}, + {"F21", 0x84, std::nullopt}, {"F22", 0x85, std::nullopt}, + {"F23", 0x86, std::nullopt}, {"F24", 0x87, std::nullopt}, + {"RightControl", 0xA3, 0x3E}, {"RightAlt", 0xA5, 0x3D}, {"RightShift", 0xA1, 0x3C}, + {"Backquote", 0xC0, 0x32}, {"Insert", 0x2D, 0x72}, {"Pause", 0x13, std::nullopt}, +}}; + +} // namespace push_to_talk_detail + +// Returns the key for a saved or typed name, or nullopt when it is unknown. +// A key the current platform cannot report is still a valid name: a profile +// moved between systems keeps it, and the interface says it is unusable. +inline std::optional parsePushToTalkKey(std::string_view name) { + // Mouse 4 and 5 are the side buttons: XBUTTON1/2, NSEvent buttons 3/4. + if (name == "Mouse4") return PushToTalkKey{PushToTalkKey::Kind::mouse, std::string(name), 1, 3}; + if (name == "Mouse5") return PushToTalkKey{PushToTalkKey::Kind::mouse, std::string(name), 2, 4}; + for (const auto& key : push_to_talk_detail::kKeys) { + if (key.name == name) { + return PushToTalkKey{PushToTalkKey::Kind::keyboard, std::string(name), key.windows, key.mac}; + } + } + return std::nullopt; +} + +} // namespace monky::light diff --git a/apps/light/src/ui/tray_controller.hpp b/apps/light/src/ui/tray_controller.hpp index eea50ac8..2c555a0b 100644 --- a/apps/light/src/ui/tray_controller.hpp +++ b/apps/light/src/ui/tray_controller.hpp @@ -64,6 +64,40 @@ class TrayController final { void reconnect() { core_.command(Json{{"command", "reconnect"}}); } + void setPushToTalk(bool enabled) { + core_.command(Json{{"command", "set-input-mode"}, + {"mode", enabled ? "push_to_talk" : "voice_activity"}}); + } + + // Interface thread, from the global hook. Key repeat and duplicate reports + // become one command per real press and release. + void pushToTalk(bool pressed) { + { + std::lock_guard lock(mutex_); + if (pressed_ == pressed) return; + pressed_ = pressed; + } + core_.command(Json{{"command", "ptt"}, {"active", pressed}}); + } + + // The hook was removed while the key may be down (call ended, mode or key + // changed), so no release will ever arrive. Leaving the call or the mode + // already released it in the core, which would refuse a release outside + // push-to-talk; a changed key would not, so it is released here. + void releasePushToTalk() { + bool release = false; + { + std::lock_guard lock(mutex_); + release = pressed_ && model_.presentation().pushToTalkChecked; + pressed_ = false; + } + if (release) core_.command(Json{{"command", "ptt"}, {"active", false}}); + } + + void setVolume(const std::string& sessionId, int volume) { + core_.command(Json{{"command", "set-volume"}, {"sessionId", sessionId}, {"volume", volume}}); + } + void quit() { core_.quit(); } private: @@ -75,6 +109,7 @@ class TrayController final { CoreCommands core_; mutable std::mutex mutex_; TrayModel model_; + bool pressed_ = false; }; } // namespace monky::light::ui diff --git a/apps/light/src/ui/tray_macos.mm b/apps/light/src/ui/tray_macos.mm index f9be89dd..290faf2e 100644 --- a/apps/light/src/ui/tray_macos.mm +++ b/apps/light/src/ui/tray_macos.mm @@ -2,8 +2,10 @@ #include "ui/tray_controller.hpp" #import +#import #import +#include #include #include #include @@ -24,6 +26,17 @@ constexpr NSInteger kJoin = 6; constexpr NSInteger kInput = 7; constexpr NSInteger kOutput = 8; +constexpr NSInteger kPushToTalk = 9; +constexpr NSInteger kVolume = 10; + +// Device-dependent modifier bits (NX_DEVICER*KEYMASK): a right-hand modifier +// arrives as flagsChanged, and only these bits tell it from the left one. +std::optional rightModifierMask(std::uint16_t code) { + if (code == 0x3C) return 0x00000004; // Right Shift + if (code == 0x3E) return 0x00002000; // Right Control + if (code == 0x3D) return 0x00000040; // Right Option + return std::nullopt; +} NSString* text(const std::string& value) { NSString* result = [NSString stringWithUTF8String:value.c_str()]; @@ -70,6 +83,7 @@ - (instancetype)initWithOwner:(void*)owner; ~MacTray() override { @autoreleasepool { + unhook(); menu_.delegate = nil; if (item_) [[NSStatusBar systemStatusBar] removeStatusItem:item_]; } @@ -108,6 +122,8 @@ int run() override { void build(NSMenu* menu) { const auto view = controller_.presentation(); + // Opening the menu is when someone returns after granting Accessibility. + watch(view.hotkey); [menu removeAllItems]; add(menu, view.status, 0, false, false, nil); // macOS notifications need an authorized, signed bundle, which belongs to @@ -128,6 +144,25 @@ void build(NSMenu* menu) { [menu addItem:[NSMenuItem separatorItem]]; add(menu, view.muteLabel, kMute, view.muteEnabled, view.muteChecked, nil); add(menu, view.deafenLabel, kDeafen, view.deafenEnabled, view.deafenChecked, nil); + add(menu, view.pushToTalkLabel, kPushToTalk, view.connected, view.pushToTalkChecked, nil); + if (!view.people.empty()) { + [menu addItem:[NSMenuItem separatorItem]]; + NSMenu* people = [[NSMenu alloc] init]; + people.autoenablesItems = NO; + for (const auto& person : view.people) { + NSMenu* volumes = [[NSMenu alloc] init]; + volumes.autoenablesItems = NO; + for (const auto preset : kVolumePresets) { + add(volumes, std::to_string(preset) + "%", kVolume, true, person.volume == preset, + @[ text(person.sessionId), @(preset) ]); + } + auto label = person.name; + if (person.volume != 100) label += " · " + std::to_string(person.volume) + "%"; + if (person.speaking) label += " (" + std::string(text_.speaking) + ")"; + submenu(people, label, volumes, true); + } + submenu(menu, text_.inThisCall, people, true); + } [menu addItem:[NSMenuItem separatorItem]]; NSMenu* devices = [[NSMenu alloc] init]; devices.autoenablesItems = NO; @@ -140,6 +175,11 @@ void build(NSMenu* menu) { } void activate(NSMenuItem* item) { + if (item.tag == kVolume) { + NSArray* choice = item.representedObject; + if (choice.count == 2) controller_.setVolume(utf8(choice[0]), [choice[1] intValue]); + return; + } const auto identifier = utf8((NSString*)item.representedObject); const auto wanted = item.state != NSControlStateValueOn; switch (item.tag) { @@ -148,6 +188,7 @@ void activate(NSMenuItem* item) { case kMute: controller_.setMuted(wanted); break; case kDeafen: controller_.setDeafened(wanted); break; case kReconnect: controller_.reconnect(); break; + case kPushToTalk: controller_.setPushToTalk(wanted); break; case kJoin: controller_.join(identifier); break; case kInput: controller_.selectInput(identifier); break; case kOutput: controller_.selectOutput(identifier); break; @@ -157,7 +198,7 @@ void activate(NSMenuItem* item) { private: void add(NSMenu* menu, const std::string& label, NSInteger tag, bool enabled, bool checked, - NSString* represented) { + id represented) { NSMenuItem* item = [[NSMenuItem alloc] initWithTitle:text(label) action:nil keyEquivalent:@""]; if (tag != 0) { item.action = @selector(activate:); @@ -189,9 +230,11 @@ void submenu(NSMenu* menu, const std::string& label, NSMenu* attached, bool enab void refresh() { const auto view = controller_.presentation(); if (auto alert = controller_.takeAlert()) alert_ = std::move(alert); + watch(view.hotkey); @autoreleasepool { NSString* symbol = !view.connected ? @"bolt.horizontal.circle" : view.silenced ? @"mic.slash.fill" + : view.speaking ? @"speaker.wave.2.circle.fill" : view.inCall ? @"waveform.circle.fill" : @"mic.fill"; NSImage* image = [NSImage imageWithSystemSymbolName:symbol @@ -211,8 +254,57 @@ void refresh() { } } + // A global monitor sees input meant for other applications. It is added only + // while push-to-talk can transmit and never consumes the event. Keys need the + // Accessibility permission; without it macOS delivers no key events at all, + // so the monitor is added again once the user grants it. + void watch(const std::optional& key) { + const bool keyboard = key && key->kind == PushToTalkKey::Kind::keyboard; + const bool trusted = !keyboard || AXIsProcessTrusted(); + const auto wanted = key ? std::optional(key->name) : std::nullopt; + if (wanted == watched_ && trusted == trusted_) return; + unhook(); + if (!key || !key->mac) return; + if (!trusted && !prompted_) { + prompted_ = true; + // Shows the system prompt once; the setting itself stays the user's. + NSDictionary* options = @{(__bridge NSString*)kAXTrustedCheckOptionPrompt : @YES}; + AXIsProcessTrustedWithOptions((__bridge CFDictionaryRef)options); + alert_ = std::string(text_.accessibilityRequired); + } + const auto code = *key->mac; + const auto modifier = rightModifierMask(code); + NSEventMask mask = keyboard ? (modifier ? NSEventMaskFlagsChanged : (NSEventMaskKeyDown | NSEventMaskKeyUp)) + : (NSEventMaskOtherMouseDown | NSEventMaskOtherMouseUp); + TrayController* controller = &controller_; + const bool mouse = !keyboard; + monitor_ = [NSEvent addGlobalMonitorForEventsMatchingMask:mask + handler:^(NSEvent* event) { + if (mouse) { + if (event.buttonNumber == code) controller->pushToTalk(event.type == NSEventTypeOtherMouseDown); + } else if (event.keyCode == code) { + if (modifier) controller->pushToTalk((event.modifierFlags & *modifier) != 0); + else if (!event.isARepeat) controller->pushToTalk(event.type == NSEventTypeKeyDown); + } + }]; + watched_ = wanted; + trusted_ = trusted; + } + + void unhook() { + if (!monitor_) return; + [NSEvent removeMonitor:monitor_]; + monitor_ = nil; + watched_.reset(); + controller_.releasePushToTalk(); + } + TrayController controller_; const Strings& text_; + id monitor_ = nil; + std::optional watched_; + bool trusted_ = false; + bool prompted_ = false; MonkyLightTrayTarget* target_ = nil; NSStatusItem* item_ = nil; NSMenu* menu_ = nil; diff --git a/apps/light/src/ui/tray_model.hpp b/apps/light/src/ui/tray_model.hpp index c94e361f..467f1b5a 100644 --- a/apps/light/src/ui/tray_model.hpp +++ b/apps/light/src/ui/tray_model.hpp @@ -1,10 +1,13 @@ #pragma once +#include "push_to_talk_key.hpp" #include "ui/tray_strings.hpp" #include #include +#include +#include #include #include #include @@ -22,6 +25,17 @@ struct MenuEntry { bool selected = false; }; +// Someone else in the current call, with the volume this profile hears them at. +struct Person { + std::string sessionId; + std::string name; + int volume = 100; + bool speaking = false; +}; + +// The volumes a native menu offers. A slider belongs to the future panels. +inline constexpr std::array kVolumePresets{0, 25, 50, 75, 100, 150, 200}; + // Everything the native interface draws, already localized. Platform code only // maps these fields onto menu items, so this is the one place where interface // behaviour is decided and the only part worth covering with unit tests. @@ -43,6 +57,14 @@ struct Presentation { bool deafenEnabled = false; std::string muteLabel; std::string deafenLabel; + // The local user is speaking in an established call. + bool speaking = false; + bool pushToTalkChecked = false; + std::string pushToTalkLabel; + // The key the interface must watch, only while push-to-talk can transmit: a + // global hook costs nothing when it is not installed. + std::optional hotkey; + std::vector people; }; // Rebuilds the interface state from the events the console consumer already @@ -61,6 +83,8 @@ class TrayModel final { if (name == "authenticating") return relink(Link::authenticating); if (name == "authenticated") { serverName_ = stringAt(event, "serverName"); + selfSession_ = stringAt(event, "sessionId"); + applyInput(event); applyChannels(event); // The list is only enumerated on demand; ask once the session is usable. refreshDevices_ = true; @@ -68,6 +92,36 @@ class TrayModel final { return true; } if (name == "state") return applyState(event); + if (name == "input-mode") { + pushToTalk_ = stringAt(event, "mode") == "push_to_talk"; + return true; + } + if (name == "push-to-talk-key") { + pushToTalkKey_ = idAt(event, "key"); + return true; + } + if (name == "participants") { + applyMembers(event); + return true; + } + if (name == "participant-volume") { + const auto session = stringAt(event, "sessionId"); + const auto volume = event.find("volume"); + for (auto& person : members_) { + if (person.sessionId == session && volume != event.end() && volume->is_number_integer()) { + person.volume = volume->get(); + } + } + // Only the menu shows volumes, and it is rebuilt when it opens. + return false; + } + if (name == "speaking") { + const auto session = stringAt(event, "sessionId"); + if (boolAt(event, "speaking")) speakers_.insert(session); + else speakers_.erase(session); + // Only the local user's speech changes the icon; the menu reads the rest when it opens. + return session == selfSession_; + } if (name == "channels-changed") { applyChannels(event); return true; @@ -79,6 +133,8 @@ class TrayModel final { if (name == "stopped") { mediaConnected_ = false; channelId_.reset(); + members_.clear(); + speakers_.clear(); return relink(Link::disconnected); } if (name == "voice-admitted") { @@ -88,6 +144,8 @@ class TrayModel final { if (name == "voice-left") { channelId_.reset(); mediaConnected_ = false; + members_.clear(); + speakers_.clear(); return true; } if (name == "voice-policy") return applyPolicy(event); @@ -155,6 +213,25 @@ class TrayModel final { : permissionMuted_ ? text_.noPermissionToSpeak : nullptr); result.deafenLabel = label(text_.deafen, serverDeafened_ ? text_.byServer : nullptr); + result.speaking = result.inCall && speakers_.contains(selfSession_); + result.pushToTalkChecked = pushToTalk_; + const auto key = pushToTalkKey_ ? parsePushToTalkKey(*pushToTalkKey_) : std::nullopt; + const bool usable = key && hotkeyUsable(*key); + result.pushToTalkLabel = label(text_.pushToTalk, nullptr); + if (!key) result.pushToTalkLabel += " (" + std::string(text_.noPushToTalkKey) + ")"; + else { + result.pushToTalkLabel += " (" + std::string(text_.key) + ": " + key->name; + if (!usable) result.pushToTalkLabel += ", " + std::string(text_.keyUnavailable); + result.pushToTalkLabel += ")"; + } + if (pushToTalk_ && usable && channelId_) result.hotkey = key; + for (const auto& member : members_) { + if (member.sessionId == selfSession_) continue; + auto person = member; + person.speaking = speakers_.contains(member.sessionId); + if (person.name.empty()) person.name = text_.participant; + result.people.push_back(std::move(person)); + } return result; } @@ -167,6 +244,40 @@ class TrayModel final { private: enum class Link { connecting, authenticating, ready, reconnecting, disconnected }; + // macOS reports no key code for some names a Windows profile may hold. + static bool hotkeyUsable(const PushToTalkKey& key) { +#ifdef __APPLE__ + return key.mac.has_value(); +#else + return key.windows != 0; +#endif + } + + void applyInput(const Json& event) { + if (event.contains("inputMode")) pushToTalk_ = stringAt(event, "inputMode") == "push_to_talk"; + if (event.contains("pushToTalkKey")) pushToTalkKey_ = idAt(event, "pushToTalkKey"); + } + + void applyMembers(const Json& event) { + const auto found = event.find("members"); + if (found == event.end() || !found->is_array()) return; + members_.clear(); + std::set present; + for (const auto& value : *found) { + if (!value.is_object()) continue; + Person person; + person.sessionId = stringAt(value, "sessionId"); + if (person.sessionId.empty()) continue; + person.name = stringAt(value, "name"); + const auto volume = value.find("volume"); + if (volume != value.end() && volume->is_number_integer()) person.volume = volume->get(); + present.insert(person.sessionId); + members_.push_back(std::move(person)); + } + // Someone who left cannot still be speaking. + std::erase_if(speakers_, [&](const std::string& session) { return !present.contains(session); }); + } + static std::string stringAt(const Json& value, const char* key) { const auto found = value.find(key); return found != value.end() && found->is_string() ? found->get() : std::string(); @@ -243,6 +354,8 @@ class TrayModel final { // mediaActive only means an engine exists; it never promotes a pending call. if (!boolAt(event, "mediaActive")) mediaConnected_ = false; applyChannels(event); + applyInput(event); + applyMembers(event); applyPolicy(event); return true; } @@ -368,6 +481,11 @@ class TrayModel final { std::optional unavailableOutput_; std::optional alert_; bool refreshDevices_ = false; + std::string selfSession_; + bool pushToTalk_ = false; + std::optional pushToTalkKey_; + std::vector members_; + std::set speakers_; }; } // namespace monky::light::ui diff --git a/apps/light/src/ui/tray_strings.hpp b/apps/light/src/ui/tray_strings.hpp index 0c7b8f20..223011e1 100644 --- a/apps/light/src/ui/tray_strings.hpp +++ b/apps/light/src/ui/tray_strings.hpp @@ -42,6 +42,16 @@ struct Strings { const char* serverKeyMissing; const char* serverKeyOfOtherServer; const char* serverKeyInvalid; + const char* pushToTalk; + const char* key; + const char* noPushToTalkKey; + const char* keyUnavailable; + const char* inThisCall; + const char* you; + const char* speaking; + const char* volume; + // macOS only lets a trusted application see keys pressed in other applications. + const char* accessibilityRequired; }; inline constexpr Strings kEnglish{ @@ -76,6 +86,15 @@ inline constexpr Strings kEnglish{ "This server presented an identity key before and now presented none. Monky Light did not join, so your sign-in is not handed to another server.", "This server presented the identity key of another server of this profile. Monky Light did not join, because a server may be relaying that one's sign-in.", "The server did not prove it holds the identity key it presented. Monky Light did not join.", + "Push-to-talk", + "key", + "no key: start with --ptt-key", + "not on this system", + "In the call", + "you", + "speaking", + "Volume", + "Push-to-talk cannot see the key yet. Allow Monky Light in System Settings › Privacy & Security › Accessibility.", }; inline constexpr Strings kPortuguese{ @@ -110,6 +129,15 @@ inline constexpr Strings kPortuguese{ "Este servidor já apresentou uma chave de identidade e agora não apresentou nenhuma. O Monky Light não entrou para não entregar o seu login a outro servidor.", "Este servidor apresentou a chave de identidade de outro servidor deste perfil. O Monky Light não entrou, porque um servidor pode estar repassando o login daquele.", "O servidor não provou ter a chave de identidade que apresentou. O Monky Light não entrou.", + "Apertar para falar", + "tecla", + "sem tecla: abra com --ptt-key", + "não existe neste sistema", + "Na chamada", + "você", + "falando", + "Volume", + "O apertar para falar ainda não vê a tecla. Permita o Monky Light em Ajustes do Sistema › Privacidade e Segurança › Acessibilidade.", }; inline constexpr const Strings& textFor(Language language) { diff --git a/apps/light/src/ui/tray_windows.cpp b/apps/light/src/ui/tray_windows.cpp index 8cf098f3..2fa5de14 100644 --- a/apps/light/src/ui/tray_windows.cpp +++ b/apps/light/src/ui/tray_windows.cpp @@ -29,15 +29,21 @@ constexpr UINT kLeave = 2; constexpr UINT kMute = 3; constexpr UINT kDeafen = 4; constexpr UINT kReconnect = 5; +constexpr UINT kPushToTalk = 6; constexpr UINT kRange = 0x0400; constexpr UINT kChannels = 0x1000; constexpr UINT kInputs = kChannels + kRange; constexpr UINT kOutputs = kInputs + kRange; +// Each person owns one row per volume preset. +constexpr UINT kPeople = kOutputs + kRange; +constexpr UINT kPresets = static_cast(kVolumePresets.size()); // The product palette, matching the accent, success and danger tokens of the // full client so both editions report the same states with the same colours. constexpr std::uint32_t kIdle = 0x5865F2; constexpr std::uint32_t kInCall = 0x23A55A; +// Lighter than the call colour, so speaking reads as the same state, active. +constexpr std::uint32_t kSpeaking = 0x57F287; constexpr std::uint32_t kSilenced = 0xF23F43; constexpr std::uint32_t kOffline = 0x8A8A8A; @@ -162,6 +168,7 @@ class WindowsTray final : public CoreInterface { } ~WindowsTray() override { + unhook(); removeIcon(); if (window_) DestroyWindow(window_); if (icon_) DestroyIcon(icon_); @@ -235,6 +242,7 @@ class WindowsTray final : public CoreInterface { return 0; } if (message == WM_DESTROY) { + unhook(); removeIcon(); window_ = nullptr; PostQuitMessage(0); @@ -279,8 +287,10 @@ class WindowsTray final : public CoreInterface { const auto view = controller_.presentation(); const auto colour = !view.connected ? kOffline : view.silenced ? kSilenced + : view.speaking ? kSpeaking : view.inCall ? kInCall : kIdle; + watch(view.hotkey); // Most events move the tooltip alone, so the bitmap is only redrawn when the // state it depicts actually changed. if (colour != colour_ || view.silenced != slashed_) { @@ -356,6 +366,24 @@ class WindowsTray final : public CoreInterface { return submenu; } + void attach(HMENU menu, const std::string& label, HMENU submenu, bool enabled) const { + MENUITEMINFOW item{}; + item.cbSize = sizeof(item); + item.fMask = MIIM_SUBMENU | MIIM_STATE | MIIM_STRING; + item.fState = static_cast(enabled ? MFS_ENABLED : MFS_DISABLED); + item.hSubMenu = submenu; + const auto wide = wideFromUtf8(label); + item.dwTypeData = const_cast(wide.c_str()); + InsertMenuItemW(menu, static_cast(GetMenuItemCount(menu)), TRUE, &item); + } + + std::string personLabel(const Person& person) const { + auto result = person.name; + if (person.volume != 100) result += " · " + std::to_string(person.volume) + "%"; + if (person.speaking) result += " (" + std::string(text_.speaking) + ")"; + return result; + } + void separator(HMENU menu) const { MENUITEMINFOW item{}; item.cbSize = sizeof(item); @@ -376,18 +404,31 @@ class WindowsTray final : public CoreInterface { separator(menu); append(menu, kMute, view.muteLabel, view.muteChecked, view.muteEnabled, false); append(menu, kDeafen, view.deafenLabel, view.deafenChecked, view.deafenEnabled, false); + append(menu, kPushToTalk, view.pushToTalkLabel, view.pushToTalkChecked, view.connected, false); + if (!view.people.empty()) { + separator(menu); + const auto people = CreatePopupMenu(); + if (people) { + for (std::size_t person = 0; person < view.people.size() && person < kRange / kPresets; ++person) { + const auto& entry = view.people[person]; + const auto volumes = CreatePopupMenu(); + if (!volumes) continue; + for (UINT preset = 0; preset < kPresets; ++preset) { + append(volumes, kPeople + static_cast(person) * kPresets + preset, + std::to_string(kVolumePresets[preset]) + "%", entry.volume == kVolumePresets[preset], true, + true); + } + attach(people, personLabel(entry), volumes, true); + } + attach(menu, text_.inThisCall, people, true); + } + } separator(menu); const auto devices = CreatePopupMenu(); if (devices) { appendRows(devices, text_.input, view.inputs, kInputs, true); appendRows(devices, text_.output, view.outputs, kOutputs, true); - MENUITEMINFOW item{}; - item.cbSize = sizeof(item); - item.fMask = MIIM_SUBMENU | MIIM_STRING; - item.hSubMenu = devices; - const auto wide = wideFromUtf8(text_.audioDevices); - item.dwTypeData = const_cast(wide.c_str()); - InsertMenuItemW(menu, static_cast(GetMenuItemCount(menu)), TRUE, &item); + attach(menu, text_.audioDevices, devices, true); } separator(menu); append(menu, kReconnect, text_.reconnectNow, false, true, false); @@ -413,7 +454,14 @@ class WindowsTray final : public CoreInterface { else if (identifier == kMute) controller_.setMuted(!view.muteChecked); else if (identifier == kDeafen) controller_.setDeafened(!view.deafenChecked); else if (identifier == kReconnect) controller_.reconnect(); - else if (identifier >= kOutputs) { + else if (identifier == kPushToTalk) controller_.setPushToTalk(!view.pushToTalkChecked); + else if (identifier >= kPeople) { + const auto person = static_cast((identifier - kPeople) / kPresets); + const auto preset = static_cast((identifier - kPeople) % kPresets); + if (person < view.people.size()) { + controller_.setVolume(view.people[person].sessionId, kVolumePresets[preset]); + } + } else if (identifier >= kOutputs) { if (const auto* entry = row(view.outputs, kOutputs)) controller_.selectOutput(entry->id); } else if (identifier >= kInputs) { if (const auto* entry = row(view.inputs, kInputs)) controller_.selectInput(entry->id); @@ -422,8 +470,67 @@ class WindowsTray final : public CoreInterface { } } + // Low-level hooks see a key pressed in any application, which a hotkey + // registration cannot (it reports no release). They run on this thread's + // message loop, are installed only while push-to-talk can transmit, and + // never swallow the input. The mouse hook is used only for a mouse button, + // since it is called on every pointer movement. + void watch(const std::optional& key) { + const auto wanted = key ? std::optional(key->name) : std::nullopt; + if (wanted == watched_) return; + unhook(); + if (!key) return; + hooked_ = this; + hotkey_ = *key; + const auto type = key->kind == PushToTalkKey::Kind::mouse ? WH_MOUSE_LL : WH_KEYBOARD_LL; + hook_ = SetWindowsHookExW(type, key->kind == PushToTalkKey::Kind::mouse ? &WindowsTray::mouse + : &WindowsTray::keyboard, + instance_, 0); + if (!hook_) { + hooked_ = nullptr; + return; + } + watched_ = wanted; + } + + void unhook() { + if (!hook_) return; + UnhookWindowsHookEx(hook_); + hook_ = nullptr; + hooked_ = nullptr; + watched_.reset(); + controller_.releasePushToTalk(); + } + + static LRESULT CALLBACK keyboard(int code, WPARAM message, LPARAM data) { + if (code == HC_ACTION && hooked_) { + const auto* key = reinterpret_cast(data); + if (key->vkCode == hooked_->hotkey_.windows) { + if (message == WM_KEYDOWN || message == WM_SYSKEYDOWN) hooked_->controller_.pushToTalk(true); + else if (message == WM_KEYUP || message == WM_SYSKEYUP) hooked_->controller_.pushToTalk(false); + } + } + return CallNextHookEx(nullptr, code, message, data); + } + + static LRESULT CALLBACK mouse(int code, WPARAM message, LPARAM data) { + if (code == HC_ACTION && hooked_ && (message == WM_XBUTTONDOWN || message == WM_XBUTTONUP)) { + const auto* event = reinterpret_cast(data); + if (HIWORD(event->mouseData) == hooked_->hotkey_.windows) { + hooked_->controller_.pushToTalk(message == WM_XBUTTONDOWN); + } + } + return CallNextHookEx(nullptr, code, message, data); + } + + // A hook procedure has no context argument; the process has one tray. + static inline WindowsTray* hooked_ = nullptr; + TrayController controller_; const Strings& text_; + HHOOK hook_ = nullptr; + PushToTalkKey hotkey_; + std::optional watched_; HINSTANCE instance_ = nullptr; HWND window_ = nullptr; HWND target_ = nullptr; diff --git a/apps/light/test/fixture_audio_device_test.cpp b/apps/light/test/fixture_audio_device_test.cpp index 536e586c..844b7e7c 100644 --- a/apps/light/test/fixture_audio_device_test.cpp +++ b/apps/light/test/fixture_audio_device_test.cpp @@ -442,6 +442,9 @@ void CheckDeviceSelection() { Require(policy->SetPolicy(true, true) == 0 && policy->StartRecording() == 0 && policy->StartPlayout() == 0 && transport.WaitFor(3, 3), "Selected devices start"); + // The synthetic tone is a 2048-amplitude sine: RMS 20*log10(1448/32768) = -27 dBov. + const auto level = policy->TakeInputLevel(); + Require(level >= 26 && level <= 28, "The policy device measures capture before forwarding it"); Require(policy->SelectDevices({std::string("mic-a"), std::string("out-b")}) == 0 && device->Recording() && device->Playing() && @@ -462,6 +465,8 @@ void CheckDeviceSelection() { "A reconnected preferred device is selected again"); Require(policy->SetPolicy(false, true) == 0 && !device->Recording(), "Mute stops capture"); + static_cast(policy->TakeInputLevel()); + Require(policy->TakeInputLevel() == media::kSilentLevel, "Stopped capture reports silence"); Require(policy->SelectDevices({std::string("mic-b"), std::string("out-b")}) == 0 && !device->Recording() && device->Snapshot().selected_input == "mic-b", "Switching the input while muted must not reopen capture"); @@ -470,6 +475,28 @@ void CheckDeviceSelection() { Require(policy->RegisterAudioCallback(nullptr) == 0, "Detach the policy transport"); } +void CheckSpeechActivity() { + const std::array silent{}; + const std::array full{32767, -32768, 32767, -32768}; + Require(media::LevelOf(silent.data(), silent.size()) == media::kSilentLevel && + media::LevelOf(full.data(), full.size()) == 0 && + media::LevelOf(full.data(), 0) == media::kSilentLevel, + "Levels use the RFC 6464 -dBov scale"); + media::SpeechDetector detector; + Require(!detector.Sample(media::kSilentLevel) && detector.Sample(media::kSpeechLevel) == true && + !detector.Sample(media::kSpeechLevel + 1) && !detector.Sample(media::kSilentLevel) && + !detector.Sample(media::kSilentLevel) && !detector.Sample(10), + "Speech starts at the threshold and survives short gaps"); + for (unsigned index = 1; index < media::kSpeechHoldSamples; ++index) { + Require(!detector.Sample(media::kSilentLevel), "Speech holds through the quiet window"); + } + Require(detector.Sample(media::kSilentLevel) == false && !detector.speaking(), + "Speech ends after the quiet window"); + Require(detector.Sample(0) == true && detector.Silence() == false && !detector.Silence() && + !detector.Sample(media::kSilentLevel), + "Silence ends speech at once and reports only one change"); +} + void CheckFailures() { Transport transport; auto device = FixtureAudioDevice::Create(); @@ -545,6 +572,7 @@ int main() { CheckInflightStopAndTerminate(false); CheckInflightStopAndTerminate(true); CheckDeviceSelection(); + CheckSpeechActivity(); CheckFailures(); CheckDestruction(); CheckTransportDestructionAfterRemoval(); diff --git a/apps/light/test/headless_interop.test.cjs b/apps/light/test/headless_interop.test.cjs index dbf1b6f5..844c622f 100644 --- a/apps/light/test/headless_interop.test.cjs +++ b/apps/light/test/headless_interop.test.cjs @@ -203,6 +203,123 @@ for (const voiceMode of ['p2p', 'sfu']) { }); } +for (const voiceMode of ['p2p', 'sfu']) { + test(`headless clients report ${voiceMode} speech and gate it with push-to-talk`, { timeout: 60_000 }, async t => { + const fixture = await createServerFixture(t, { voiceMode }); + const alice = new NativeClient(fixture, { nickname: 'Light Alice' }); + const bob = new NativeClient(fixture, { nickname: 'Light Bob' }); + const auth = await alice.wait('authenticated'); + await bob.wait('authenticated'); + const channelId = auth.channels.find(channel => channel.type === 'VOICE').id; + await alice.join(channelId); + await bob.join(channelId); + // The synthetic microphone plays a constant tone, so both sides see Alice speak. + // Without an action the change may already have happened: Alice speaks as + // soon as her own call starts, possibly before Bob's join returns. + const speech = async (speaking, action) => { + const since = action ? [alice.events.length, bob.events.length] : [0, 0]; + if (action) await action(); + await Promise.all([alice, bob].map((client, index) => client.wait(event => event.event === 'speaking' && + event.sessionId === auth.sessionId && event.speaking === speaking, since[index]))); + }; + await speech(true); + + // An interface starts from the authenticated event and follows the roster by events. + assert.equal(auth.inputMode, 'voice_activity'); + assert.equal(auth.pushToTalkKey, null); + const bobAuth = bob.events.find(event => event.event === 'authenticated'); + const roster = await alice.wait(event => event.event === 'participants' && event.members.length === 2); + assert.deepEqual(roster.members.map(member => [member.sessionId, member.name, member.self]).sort(), + [[auth.sessionId, 'Light Alice', true], [bobAuth.sessionId, 'Light Bob', false]].sort()); + assert.equal((await alice.state()).members.length, 2); + const keySince = alice.events.length; + assert.equal((await alice.command('set-ptt-key', { key: 'F13' })).event, 'command-accepted'); + await alice.wait(event => event.event === 'push-to-talk-key' && event.key === 'F13', keySince); + assert.equal((await alice.state()).pushToTalkKey, 'F13'); + for (const key of ['CapsLock', 'f13', 7]) { + assert.equal((await alice.command('set-ptt-key', { key })).event, 'command-error', `Key ${key} must be refused`); + } + + assert.equal((await alice.command('ptt', { active: true })).event, 'command-error', + 'Push-to-talk must not be accepted in voice-activity mode'); + const modeSince = alice.events.length; + await speech(false, () => alice.command('set-input-mode', { mode: 'push_to_talk' })); + await alice.wait(event => event.event === 'input-mode' && event.mode === 'push_to_talk', modeSince); + const gated = await alice.state(); + assert.equal(gated.inputMode, 'push_to_talk'); + assert.equal(gated.pushToTalkActive, false); + assert.equal(gated.audioDevice.recording, true, 'Push-to-talk keeps capture open so a press is immediate'); + + await speech(true, () => alice.command('ptt', { active: true })); + await speech(false, () => alice.command('ptt', { active: false })); + await speech(true, () => alice.command('ptt', { active: true })); + await speech(false, () => alice.command('mute', { enabled: true })); + await speech(true, () => alice.command('mute', { enabled: false })); + await alice.command('set-input-mode', { mode: 'voice_activity' }); + const restored = await alice.state(); + assert.equal(restored.inputMode, 'voice_activity'); + assert.equal(restored.pushToTalkActive, false, 'Switching modes releases a held key'); + await alice.close(); + await bob.close(); + }); + + test(`participant volume scales ${voiceMode} playback and is kept in the profile`, { timeout: 60_000 }, async t => { + const fixture = await createServerFixture(t, { voiceMode }); + const alice = new NativeClient(fixture, { nickname: 'Light Alice' }); + const bob = new NativeClient(fixture, { nickname: 'Light Bob', muted: true }); + const aliceAuth = await alice.wait('authenticated'); + await bob.wait('authenticated'); + const channelId = aliceAuth.channels.find(channel => channel.type === 'VOICE').id; + await alice.join(channelId); + await bob.join(channelId); + // Only Alice transmits, so Bob's playout is exactly her scaled stream. + await bob.untilState(value => value.audioDevice.nonzeroPlayoutCallbacks >= 25, 'Bob did not hear Alice'); + const window = async () => { + await new Promise(resolve => setTimeout(resolve, 400)); + const start = (await bob.state()).audioDevice; + await new Promise(resolve => setTimeout(resolve, 1000)); + const end = (await bob.state()).audioDevice; + const callbacks = end.playoutCallbacks - start.playoutCallbacks; + return { nonzero: end.nonzeroPlayoutCallbacks - start.nonzeroPlayoutCallbacks, + energy: (end.outputEnergy - start.outputEnergy) / callbacks }; + }; + const normal = await window(); + const setVolume = async volume => { + const since = bob.events.length; + assert.equal((await bob.command('set-volume', { sessionId: aliceAuth.sessionId, volume })).event, + 'command-accepted'); + await bob.wait(event => event.event === 'participant-volume' && event.sessionId === aliceAuth.sessionId && + event.volume === volume, since); + }; + await setVolume(0); + assert.ok((await window()).nonzero <= 2, 'Volume 0 must silence that participant'); + await setVolume(200); + const loud = await window(); + assert.ok(loud.energy > normal.energy * 2.5, + `Volume 200 must amplify (${loud.energy} vs ${normal.energy})`); + assert.equal((await bob.state()).userVolumes[aliceAuth.sessionId], 200); + for (const invalid of [{ volume: 201 }, { volume: 1.5 }, { volume: -1 }]) { + assert.equal((await bob.command('set-volume', { sessionId: aliceAuth.sessionId, ...invalid })).event, + 'command-error'); + } + + // A new call, even after a restart, applies the saved volume from the start. + await bob.close(); + const again = new NativeClient(fixture, { nickname: 'Light Bob', muted: true, profile: bob.profile, + pushToTalkKey: 'Mouse5' }); + assert.equal((await again.wait('authenticated')).pushToTalkKey, 'Mouse5', '--ptt-key must be saved and reported'); + await again.command('set-volume', { sessionId: aliceAuth.sessionId, volume: 0 }); + await again.join(channelId); + await again.untilState(value => value.userVolumes[aliceAuth.sessionId] === 0 && value.audioDevice.playoutCallbacks > 100, + 'Saved volume was not reported for the new call'); + assert.ok((await again.state()).audioDevice.nonzeroPlayoutCallbacks <= 2, 'Saved volume 0 was not applied on join'); + await again.command('set-volume', { sessionId: aliceAuth.sessionId, volume: 100 }); + assert.deepEqual((await again.state()).userVolumes, {}, 'Volume 100 must free its stored entry'); + await again.close(); + await alice.close(); + }); +} + test('native calls survive both live P2P/SFU topology transitions and preserve moderation', { timeout: 60_000 }, async t => { const fixture = await createServerFixture(t); const admin = await fixture.connectHuman('Fixture admin'); diff --git a/apps/light/test/native_client.cjs b/apps/light/test/native_client.cjs index 5022a587..5d9b0bd3 100644 --- a/apps/light/test/native_client.cjs +++ b/apps/light/test/native_client.cjs @@ -13,7 +13,7 @@ class NativeClient { ended = false; expectedPeerFailures = new Set(); - constructor(fixture, { nickname, profile, password, server, synthetic = true, microphoneAccess, defaultProcessing = false, muted = false, deafened = false, tray = false, pseudoConsole = false } = {}) { + constructor(fixture, { nickname, profile, password, server, synthetic = true, microphoneAccess, defaultProcessing = false, muted = false, deafened = false, tray = false, pseudoConsole = false, pushToTalkKey } = {}) { this.nickname = nickname; this.profile = profile ?? path.join(fixture.directory, `native-${randomUUID()}`); const env = { ...process.env }; @@ -32,6 +32,7 @@ class NativeClient { } if (muted) args.push('--muted'); if (deafened) args.push('--deafened'); + if (pushToTalkKey) args.push('--ptt-key', pushToTalkKey); if (tray) { // The fixture's interface has the tray's threads, not its icon. assert.ok(synthetic, 'The real tray would show an icon; tests use the fixture interface'); diff --git a/apps/light/test/profile_identity_test.cpp b/apps/light/test/profile_identity_test.cpp index b5af5bcd..47fea23a 100644 --- a/apps/light/test/profile_identity_test.cpp +++ b/apps/light/test/profile_identity_test.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -399,9 +400,11 @@ void settingsBesideIdentity(const fs::path& root) { ProfileIdentity identity(profile.path); publicKey = identity.publicKeyHex(); const auto defaults = identity.loadSettings(); - require(!defaults.inputDeviceId && !defaults.outputDeviceId, "Missing settings did not yield defaults"); + require(!defaults.inputDeviceId && !defaults.outputDeviceId && !defaults.pushToTalk, + "Missing settings did not yield defaults"); identity.saveSettings({std::string("input-\xc3\xa1"), std::nullopt}); - identity.saveSettings({std::string("input-\xc3\xa1"), std::string("output")}); + identity.saveSettings({std::string("input-\xc3\xa1"), std::string("output"), true, + {{"user:device-a", 0}, {"user:device-b", 200}}, std::string("RightControl")}); require(!fs::exists(profile.path / "monky-light-settings.json.pending"), "Settings write left its marker"); requireFailure([&] { identity.saveSettings({std::string(), std::nullopt}); }, "An empty device ID was saved"); @@ -409,6 +412,20 @@ void settingsBesideIdentity(const fs::path& root) { "An unbounded device ID was saved"); requireFailure([&] { identity.saveSettings({std::string("\xff"), std::nullopt}); }, "An invalid UTF-8 device ID was saved"); + requireFailure([&] { identity.saveSettings({std::nullopt, std::nullopt, false, {{"session", 201}}}); }, + "An out-of-range participant volume was saved"); + requireFailure([&] { identity.saveSettings({std::nullopt, std::nullopt, false, {{"session", 100}}}); }, + "The default participant volume was stored"); + requireFailure([&] { identity.saveSettings({std::nullopt, std::nullopt, false, {{std::string(257, 's'), 50}}}); }, + "An unbounded session ID was saved"); + std::map crowded; + for (std::size_t index = 0; index <= monky::light::kMaxUserVolumes; ++index) { + crowded.emplace("session-" + std::to_string(index), 50); + } + requireFailure([&] { identity.saveSettings({std::nullopt, std::nullopt, false, crowded}); }, + "Unbounded participant volumes were saved"); + requireFailure([&] { identity.saveSettings({std::nullopt, std::nullopt, false, {}, std::string("CapsLock")}); }, + "An unknown push-to-talk key was saved"); } const auto saved = readFile(settingsPath); // An interrupted settings write is not identity state and must not block the profile. @@ -418,15 +435,35 @@ void settingsBesideIdentity(const fs::path& root) { require(identity.publicKeyHex() == publicKey, "Settings changed the profile identity"); const auto loaded = identity.loadSettings(); require(loaded.inputDeviceId == std::optional("input-\xc3\xa1") && - loaded.outputDeviceId == std::optional("output"), "Settings did not round-trip"); + loaded.outputDeviceId == std::optional("output") && loaded.pushToTalk && + loaded.userVolumes == std::map{{"user:device-a", 0}, {"user:device-b", 200}} && + loaded.pushToTalkKey == std::optional("RightControl"), + "Settings did not round-trip"); identity.saveSettings({std::nullopt, std::string("output")}); require(!fs::exists(profile.path / "monky-light-settings.json.pending"), "Settings write did not replace an interrupted marker"); } + // Profiles saved before push-to-talk existed keep voice activity. + writeFile(settingsPath, "{\"format\":\"monky-light-settings\",\"version\":1,\"inputDeviceId\":\"mic\"}"); + { + ProfileIdentity identity(profile.path); + const auto legacy = identity.loadSettings(); + require(legacy.inputDeviceId == std::optional("mic") && !legacy.pushToTalk, + "Settings without an input mode did not default to voice activity"); + } for (const auto* invalid : {"", "{", "[]", "{\"format\":\"monky-light-settings\",\"version\":2}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"inputMode\":\"always\"}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"inputMode\":true}", "{\"format\":\"other\",\"version\":1}", "{\"format\":\"monky-light-settings\",\"version\":1,\"inputDeviceId\":7}", - "{\"format\":\"monky-light-settings\",\"version\":1,\"inputDeviceId\":{}}"}) { + "{\"format\":\"monky-light-settings\",\"version\":1,\"inputDeviceId\":{}}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"userVolumes\":[]}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"userVolumes\":{\"s\":201}}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"userVolumes\":{\"s\":\"50\"}}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"userVolumes\":{\"s\":{}}}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"other\":{}}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"pushToTalkKey\":\"Esc\"}", + "{\"format\":\"monky-light-settings\",\"version\":1,\"pushToTalkKey\":13}"}) { writeFile(settingsPath, invalid); ProfileIdentity identity(profile.path); require(identity.publicKeyHex() == publicKey, "Invalid settings affected identity validation"); diff --git a/apps/light/test/tray_model_test.cpp b/apps/light/test/tray_model_test.cpp index 1f756f62..17a88015 100644 --- a/apps/light/test/tray_model_test.cpp +++ b/apps/light/test/tray_model_test.cpp @@ -325,6 +325,123 @@ void theControllerSendsConsoleCommands() { require(quit, "Quit must reach the core"); } +Json members() { + return {{"event", "participants"}, + {"members", Json::array({{{"sessionId", "user:a"}, {"name", "Me"}, {"self", true}, {"volume", 100}}, + {{"sessionId", "ana:b"}, {"name", "Ana"}, {"self", false}, {"volume", 150}}, + {{"sessionId", "bia:c"}, {"name", ""}, {"self", false}, {"volume", 100}}})}}; +} + +void pushToTalkWatchesTheKeyOnlyWhileItCanTransmit() { + TrayModel model(Language::english); + auto start = authenticated(); + start["inputMode"] = "push_to_talk"; + start["pushToTalkKey"] = nullptr; + model.apply(start); + auto view = model.presentation(); + require(view.pushToTalkChecked, "The saved input mode must reach the interface at start"); + require(view.pushToTalkLabel == "Push-to-talk (no key: start with --ptt-key)", + "Without a key the menu must say how to choose one"); + require(!view.hotkey, "No key means nothing to watch"); + + require(model.apply({{"event", "push-to-talk-key"}, {"key", "F13"}}), "A new key must redraw"); + require(model.presentation().pushToTalkLabel == "Push-to-talk (key: F13)", "The menu must name the key"); + require(!model.presentation().hotkey, "Outside a call the key must not be watched"); + + model.apply({{"event", "voice-admitted"}, {"channelId", "first"}}); + view = model.presentation(); + require(view.hotkey && view.hotkey->name == "F13" && view.hotkey->windows == 0x7C, + "In a call the key must be watched with its platform code"); + require(model.apply({{"event", "input-mode"}, {"mode", "voice_activity"}}), "A mode change must redraw"); + require(!model.presentation().hotkey && !model.presentation().pushToTalkChecked, + "Voice activity must stop watching the key"); + model.apply({{"event", "input-mode"}, {"mode", "push_to_talk"}}); + model.apply({{"event", "push-to-talk-key"}, {"key", "Mouse4"}}); + view = model.presentation(); + require(view.hotkey && view.hotkey->kind == monky::light::PushToTalkKey::Kind::mouse, + "A side button must be watched as a mouse button"); + model.apply({{"event", "voice-left"}}); + require(!model.presentation().hotkey, "Leaving the call must stop watching the key"); +} + +void peopleCarrySpeechAndVolume() { + TrayModel model(Language::english); + model.apply(authenticated()); + model.apply({{"event", "voice-admitted"}, {"channelId", "first"}}); + model.apply({{"event", "media-connected"}}); + require(model.apply(members()), "A new roster must redraw"); + auto view = model.presentation(); + require(view.people.size() == 2, "The local user is not listed among the people to adjust"); + require(view.people[0].name == "Ana" && view.people[0].volume == 150, "A saved volume must be shown"); + require(view.people[1].name == "participant", "A nameless participant still needs a label"); + + require(!model.apply({{"event", "speaking"}, {"sessionId", "ana:b"}, {"speaking", true}}), + "Someone else's speech must not wake the interface thread"); + require(model.presentation().people[0].speaking, "The menu must show who is speaking"); + require(!model.presentation().speaking, "Someone else speaking is not the local user speaking"); + require(model.apply({{"event", "speaking"}, {"sessionId", "user:a"}, {"speaking", true}}), + "The local user's speech changes the icon"); + require(model.presentation().speaking, "The icon must show the local user speaking"); + + require(!model.apply({{"event", "participant-volume"}, {"sessionId", "ana:b"}, {"volume", 0}}), + "A volume only appears in the menu, which is rebuilt when it opens"); + require(model.presentation().people[0].volume == 0, "A changed volume must be shown"); + + auto remaining = members(); + remaining["members"].erase(1); + model.apply(remaining); + model.apply(members()); + require(!model.presentation().people[0].speaking, "Someone who left the call must stop speaking"); + model.apply({{"event", "voice-left"}}); + view = model.presentation(); + require(view.people.empty() && !view.speaking, "Leaving the call clears the people and the speech"); +} + +void theControllerGatesPushToTalk() { + std::vector sent; + TrayController controller({[&](Json value) { sent.push_back(std::move(value)); }, [] {}}, Language::english); + auto start = authenticated(); + start["inputMode"] = "push_to_talk"; + controller.observe(start); + sent.clear(); + controller.pushToTalk(true); + controller.pushToTalk(true); + require(sent.size() == 1 && sent.back().at("command") == "ptt" && sent.back().at("active") == true, + "Key repeat must become a single press"); + controller.pushToTalk(false); + require(sent.size() == 2 && sent.back().at("active") == false, "A release must be sent once"); + controller.releasePushToTalk(); + require(sent.size() == 2, "Nothing is held, so removing the hook sends nothing"); + controller.pushToTalk(true); + controller.releasePushToTalk(); + require(sent.size() == 4 && sent.back().at("active") == false, + "A key held while its hook is removed must still be released"); + controller.observe({{"event", "input-mode"}, {"mode", "voice_activity"}}); + controller.pushToTalk(true); + controller.releasePushToTalk(); + require(sent.size() == 5, "Outside push-to-talk the core already released it and would refuse"); + + controller.setPushToTalk(true); + require(sent.back().at("command") == "set-input-mode" && sent.back().at("mode") == "push_to_talk", + "The menu toggle must change the input mode"); + controller.setVolume("ana:b", 50); + require(sent.back().at("command") == "set-volume" && sent.back().at("sessionId") == "ana:b" && + sent.back().at("volume") == 50, + "A volume row must carry its session and preset"); +} + +void pushToTalkKeysAreNamedForBothPlatforms() { + using monky::light::parsePushToTalkKey; + require(!parsePushToTalkKey("CapsLock") && !parsePushToTalkKey("f13") && !parsePushToTalkKey(""), + "Unknown, lower-case and toggling keys are refused"); + const auto letter = parsePushToTalkKey("A"); + require(letter && letter->windows == 0x41 && letter->mac == 0, "kVK_ANSI_A is zero, not missing"); + const auto modifier = parsePushToTalkKey("RightControl"); + require(modifier && modifier->windows == 0xA3 && modifier->mac == 0x3E, "Right Control maps on both systems"); + const auto windowsOnly = parsePushToTalkKey("F24"); + require(windowsOnly && !windowsOnly->mac, "macOS cannot report F24"); +} + void malformedEventsAreIgnored() { TrayModel model(Language::english); require(!model.apply(Json::array({1, 2})), "A non-object event must be ignored"); @@ -347,6 +464,10 @@ int main() { failuresAndPermissionsReachTheInterface(); portugueseFollowsTheSameStates(); theControllerSendsConsoleCommands(); + pushToTalkWatchesTheKeyOnlyWhileItCanTransmit(); + peopleCarrySpeechAndVolume(); + theControllerGatesPushToTalk(); + pushToTalkKeysAreNamedForBothPlatforms(); malformedEventsAreIgnored(); std::cout << "Tray interface scenarios passed\n"; return 0; diff --git a/release-notes/658-light-participant-volume.json b/release-notes/658-light-participant-volume.json new file mode 100644 index 00000000..537876d1 --- /dev/null +++ b/release-notes/658-light-participant-volume.json @@ -0,0 +1,5 @@ +{ + "group": "novidades", + "pt-BR": "O núcleo do Monky Light agora ajusta de 0 a 200% o volume de cada participante da chamada, como o controle do aplicativo completo, e lembra a escolha nas próximas chamadas.", + "en": "The Monky Light core can now set each call participant's volume from 0 to 200%, like the full app's slider, and remembers the choice in later calls." +} diff --git a/release-notes/658-light-speaking-ptt.json b/release-notes/658-light-speaking-ptt.json new file mode 100644 index 00000000..b4813341 --- /dev/null +++ b/release-notes/658-light-speaking-ptt.json @@ -0,0 +1,5 @@ +{ + "group": "novidades", + "pt-BR": "O núcleo do Monky Light agora indica quem está falando na chamada, inclusive você, e ganhou o modo push-to-talk: a escolha fica salva no perfil, e o microfone só transmite enquanto a tecla está pressionada.", + "en": "The Monky Light core now shows who is speaking in the call, including you, and gained a push-to-talk mode: the choice is saved in the profile, and the microphone only transmits while the key is held." +} diff --git a/release-notes/658-light-tray-ptt-volume.json b/release-notes/658-light-tray-ptt-volume.json new file mode 100644 index 00000000..aaa2e4ca --- /dev/null +++ b/release-notes/658-light-tray-ptt-volume.json @@ -0,0 +1,5 @@ +{ + "group": "novidades", + "pt-BR": "A bandeja do Monky Light agora tem apertar para falar com uma tecla ou botão do mouse escolhido com --ptt-key, mostra quando você está falando e permite ajustar o volume de cada participante da chamada pelo menu.", + "en": "The Monky Light tray now has push-to-talk on a key or mouse button chosen with --ptt-key, shows when you are speaking, and lets you set each call participant's volume from the menu." +}