diff --git a/CMakeLists.txt b/CMakeLists.txt index e923a893167..76f2d19ba5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,11 +227,19 @@ set(BUILD_INSTALLER_OPTION ON) set(DISABLE_QML_OPTION OFF) set(DOWNLOAD_SERVERLESS_CONTENT_OPTION OFF) set(ENABLE_WEBRTC_DATA_CHANNELS_OPTION OFF) +set(ENABLE_WEBRTC_AUDIO_OPTION OFF) if (WIN32 OR (UNIX AND NOT APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")) set(ENABLE_WEBRTC_DATA_CHANNELS_OPTION ON) endif() +if (WIN32 OR (UNIX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")) + # We don't yet have a working libwebrtc for android. + # WebRTC is difficult to build on aarch64 Linux. + # an alternative is https://gitlab.freedesktop.org/pulseaudio/webrtc-audio-processing. + set(ENABLE_WEBRTC_AUDIO_OPTION ON) +endif() + if (ANDROID OR UWP) set(BUILD_SERVER_OPTION OFF) set(BUILD_TOOLS_OPTION OFF) @@ -304,6 +312,7 @@ option( ${DOWNLOAD_SERVERLESS_CONTENT_OPTION} ) option(ENABLE_WEBRTC_DATA_CHANNELS "Use WebRTC for client-server connection in parallel with UDP" ${ENABLE_WEBRTC_DATA_CHANNELS_OPTION}) +option(ENABLE_WEBRTC_AUDIO "Use WebRTC for audio processing (e.g. echo cancelation)." ${ENABLE_WEBRTC_AUDIO_OPTION}) set(PLATFORM_QT_GL OpenGL) @@ -311,6 +320,10 @@ if (ENABLE_WEBRTC_DATA_CHANNELS) add_compile_definitions(WEBRTC_DATA_CHANNELS=1) endif() +if (ENABLE_WEBRTC_AUDIO) + add_compile_definitions(WEBRTC_AUDIO=1) +endif() + if (USE_KHR_ROBUSTNESS) add_definitions(-DUSE_KHR_ROBUSTNESS) endif() @@ -338,6 +351,7 @@ MESSAGE(STATUS "Build installer: " ${BUILD_INSTALLER}) MESSAGE(STATUS "GL ES: " ${USE_GLES}) MESSAGE(STATUS "DL serverless content: " ${DOWNLOAD_SERVERLESS_CONTENT}) MESSAGE(STATUS "WebRTC Data Channels: " ${ENABLE_WEBRTC_DATA_CHANNELS}) +MESSAGE(STATUS "WebRTC Audio: " ${ENABLE_WEBRTC_AUDIO}) MESSAGE(STATUS "Static standard libraries: " ${STATIC_STDLIB}) MESSAGE(STATUS "Shared internal libraries: " ${BUILD_SHARED_LIBS}) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 4b9b2d5095f..cd4c28cf0f9 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -475,11 +475,6 @@ void Agent::executeScript() { packetType = PacketType::SilentAudioFrame; } - Transform audioTransform; - auto headOrientation = scriptedAvatar->getHeadOrientation(); - audioTransform.setTranslation(scriptedAvatar->getWorldPosition()); - audioTransform.setRotation(headOrientation); - QByteArray encodedBuffer; if (_encoder) { _encoder->encode(audio, encodedBuffer); @@ -488,7 +483,7 @@ void Agent::executeScript() { } AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber, false, - audioTransform, scriptedAvatar->getWorldPosition(), glm::vec3(0), + {scriptedAvatar->getWorldPosition(), scriptedAvatar->getHeadOrientation()}, scriptedAvatar->getWorldPosition(), glm::vec3(0), packetType, _selectedCodecName); }); @@ -719,7 +714,7 @@ void Agent::processAgentAvatarAudio() { if (isPlayingRecording && !_shouldMuteRecordingAudio) { _shouldMuteRecordingAudio = true; } - + auto audioData = _avatarSound->getAudioData(); nextSoundOutput = reinterpret_cast(audioData->rawData() + _numAvatarSoundSentBytes); @@ -880,7 +875,7 @@ void Agent::aboutToFinish() { { DependencyManager::get()->shutdownScripting(); } - + DependencyManager::destroy(); DependencyManager::destroy(); diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index 204095ab940..76020aa4ef7 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -279,7 +279,7 @@ bool shouldBeSkipped(MixableStream& stream, const Node& listener, bool shouldCheckIgnoreBox = (listenerAudioStream.isIgnoreBoxEnabled() || stream.positionalStream->isIgnoreBoxEnabled()); if (shouldCheckIgnoreBox && - listenerAudioStream.getIgnoreBox().touches(stream.positionalStream->getIgnoreBox())) { + isTouching(listenerAudioStream.getIgnoreBox(), stream.positionalStream->getIgnoreBox())) { return true; } @@ -587,7 +587,7 @@ void AudioMixerSlave::updateHRTFParameters(AudioMixerClientData::MixableStream& glm::vec3 relativePosition = streamToAdd->getPosition() - listeningNodeStream.getPosition(); float distance = glm::max(glm::length(relativePosition), EPSILON); - float gain = isEcho ? 1.0f : computeGain(masterAvatarGain, masterInjectorGain, listeningNodeStream, *streamToAdd, + float gain = isEcho ? 1.0f : computeGain(masterAvatarGain, masterInjectorGain, listeningNodeStream, *streamToAdd, relativePosition, distance); float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition); @@ -815,14 +815,14 @@ float computeAzimuth(const AvatarAudioStream& listeningNodeStream, float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition); if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) { - + // produce an oriented angle about the y-axis glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2)); float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward" return (direction.x < 0.0f) ? -angle : angle; - } else { + } else { // no azimuth if they are in same spot - return 0.0f; + return 0.0f; } } diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 30766b892bd..ca15de1afac 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -7387,6 +7387,7 @@ void Application::nodeActivated(SharedNodePointer node) { } } + // FIXME: this will be handled in AudioClient, once it's derived from AudioPacketHandler if (node->getType() == NodeType::AudioMixer && !isInterstitialMode()) { DependencyManager::get()->negotiateAudioFormat(); } diff --git a/libraries/audio-client-core/CMakeLists.txt b/libraries/audio-client-core/CMakeLists.txt new file mode 100644 index 00000000000..9dafc504d2f --- /dev/null +++ b/libraries/audio-client-core/CMakeLists.txt @@ -0,0 +1,22 @@ +set(TARGET_NAME audio-client-core) +if (ANDROID) + set(PLATFORM_QT_COMPONENTS AndroidExtras) +endif () +setup_hifi_library(Network) +link_hifi_libraries(audio) +include_hifi_library_headers(shared) +include_hifi_library_headers(networking) + +if (ENABLE_WEBRTC_AUDIO) + target_webrtc() +endif () + +# append audio includes to our list of includes to bubble +target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src") + +# have CMake grab externals for us +if (APPLE) + find_library(CoreAudio CoreAudio) + find_library(CoreFoundation CoreFoundation) + target_link_libraries(${TARGET_NAME} ${CoreAudio} ${CoreFoundation}) +endif () diff --git a/libraries/audio-client-core/src/AudioClientLogging.cpp b/libraries/audio-client-core/src/AudioClientLogging.cpp new file mode 100644 index 00000000000..f497e84c4cb --- /dev/null +++ b/libraries/audio-client-core/src/AudioClientLogging.cpp @@ -0,0 +1,14 @@ +// +// AudioClientLogging.cpp +// libraries/audio-client/src +// +// Created by Seth Alves on 4/6/15. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioClientLogging.h" + +Q_LOGGING_CATEGORY(audioclient, "hifi.audioclient") diff --git a/libraries/audio-client-core/src/AudioClientLogging.h b/libraries/audio-client-core/src/AudioClientLogging.h new file mode 100644 index 00000000000..462f8108ac8 --- /dev/null +++ b/libraries/audio-client-core/src/AudioClientLogging.h @@ -0,0 +1,20 @@ +// +// AudioClientLogging.h +// libraries/audio-client/src +// +// Created by Seth Alves on 4/6/15. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioClientLogging_h +#define hifi_AudioClientLogging_h + +#include + +Q_DECLARE_LOGGING_CATEGORY(audioclient) + +#endif // hifi_AudioClientLogging_h + diff --git a/libraries/audio-client-core/src/AudioIOStats.cpp b/libraries/audio-client-core/src/AudioIOStats.cpp new file mode 100644 index 00000000000..ad1057c8498 --- /dev/null +++ b/libraries/audio-client-core/src/AudioIOStats.cpp @@ -0,0 +1,209 @@ +// +// AudioStats.cpp +// interface/src/audio +// +// Created by Stephen Birarda on 2014-12-16. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioIOStats.h" + +#include +#include +#include +#include + +// This is called 1x/sec (see AudioClient) and we want it to log the last 5s +static const int INPUT_READS_WINDOW = 5; +static const int INPUT_UNPLAYED_WINDOW = 5; +static const int OUTPUT_UNPLAYED_WINDOW = 5; + +static const int APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS = (int)(30.0f * 1000.0f / AudioConstants::NETWORK_FRAME_MSECS); + + +AudioIOStats::AudioIOStats(MixedProcessedAudioStream* receivedAudioStream) : + _interface(new AudioStatsInterface(this)), + _inputMsRead(1, INPUT_READS_WINDOW), + _inputMsUnplayed(1, INPUT_UNPLAYED_WINDOW), + _outputMsUnplayed(1, OUTPUT_UNPLAYED_WINDOW), + _lastSentPacketTime(0), + _packetTimegaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS), + _receivedAudioStream(receivedAudioStream) +{ + +} + +void AudioIOStats::reset() { + _receivedAudioStream->resetStats(); + + _inputMsRead.reset(); + _inputMsUnplayed.reset(); + _outputMsUnplayed.reset(); + _packetTimegaps.reset(); + + _interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps); + _interface->updateMixerStream(AudioStreamStats()); + _interface->updateClientStream(AudioStreamStats()); + _interface->updateInjectorStreams(QHash()); +} + +void AudioIOStats::sentPacket() const { + // first time this is 0 + if (_lastSentPacketTime == 0) { + _lastSentPacketTime = usecTimestampNow(); + } else { + quint64 now = usecTimestampNow(); + quint64 gap = now - _lastSentPacketTime; + _lastSentPacketTime = now; + _packetTimegaps.update(gap); + } +} + +void AudioIOStats::processStreamStatsPacket(QSharedPointer message, SharedNodePointer sendingNode) { + // parse the appendFlag, clear injected audio stream stats if 0 + quint8 appendFlag; + message->readPrimitive(&appendFlag); + + if (appendFlag & AudioStreamStats::START) { + _injectorStreams.clear(); + } + + // parse the number of stream stats structs to follow + quint16 numStreamStats; + message->readPrimitive(&numStreamStats); + + // parse the stream stats + AudioStreamStats streamStats; + for (quint16 i = 0; i < numStreamStats; i++) { + message->readPrimitive(&streamStats); + + if (streamStats._streamType == PositionalAudioStream::Microphone) { + _interface->updateMixerStream(streamStats); + } else { + _injectorStreams[streamStats._streamIdentifier] = streamStats; + } + } + + if (appendFlag & AudioStreamStats::END) { + _interface->updateInjectorStreams(_injectorStreams); + } +} + +void AudioIOStats::publish() { + // call _receivedAudioStream's per-second callback + _receivedAudioStream->perSecondCallbackForUpdatingStats(); + + auto nodeList = DependencyManager::get(); + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + if (!audioMixer) { + return; + } + + quint8 appendFlag = AudioStreamStats::START | AudioStreamStats::END; + quint16 numStreamStatsToPack = 1; + AudioStreamStats stats = _receivedAudioStream->getAudioStreamStats(); + + // update the interface + _interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps); + _interface->updateClientStream(stats); + + // prepare a packet to the mixer + int statsPacketSize = sizeof(appendFlag) + sizeof(numStreamStatsToPack) + sizeof(stats); + auto statsPacket = NLPacket::create(PacketType::AudioStreamStats, statsPacketSize); + + // pack append flag + statsPacket->writePrimitive(appendFlag); + + // pack number of stats packed + statsPacket->writePrimitive(numStreamStatsToPack); + + // pack downstream audio stream stats + statsPacket->writePrimitive(stats); + + // send packet + nodeList->sendPacket(std::move(statsPacket), *audioMixer); +} + +AudioStreamStatsInterface::AudioStreamStatsInterface(QObject* parent) : + QObject(parent) {} + +void AudioStreamStatsInterface::updateStream(const AudioStreamStats& stats) { + lossRate(stats._packetStreamStats.getLostRate()); + lossCount(stats._packetStreamStats._lost); + lossRateWindow(stats._packetStreamWindowStats.getLostRate()); + lossCountWindow(stats._packetStreamWindowStats._lost); + + framesDesired(stats._desiredJitterBufferFrames); + framesAvailable(stats._framesAvailable); + framesAvailableAvg(stats._framesAvailableAverage); + + unplayedMsMax(stats._unplayedMs); + + starveCount(stats._starveCount); + lastStarveDurationCount(stats._consecutiveNotMixedCount); + dropCount(stats._framesDropped); + overflowCount(stats._overflowCount); + + timegapMsMax(stats._timeGapMax / USECS_PER_MSEC); + timegapMsAvg(stats._timeGapAverage / USECS_PER_MSEC); + timegapMsMaxWindow(stats._timeGapWindowMax / USECS_PER_MSEC); + timegapMsAvgWindow(stats._timeGapWindowAverage / USECS_PER_MSEC); +} + +AudioStatsInterface::AudioStatsInterface(QObject* parent) : + QObject(parent), + _client(new AudioStreamStatsInterface(this)), + _mixer(new AudioStreamStatsInterface(this)), + _injectors(new QObject(this)) {} + + +void AudioStatsInterface::updateLocalBuffers(const MovingMinMaxAvg& inputMsRead, + const MovingMinMaxAvg& inputMsUnplayed, + const MovingMinMaxAvg& outputMsUnplayed, + const MovingMinMaxAvg& timegaps) { + if (SharedNodePointer audioNode = DependencyManager::get()->soloNodeOfType(NodeType::AudioMixer)) { + pingMs(audioNode->getPingMs()); + } + + inputReadMsMax(inputMsRead.getWindowMax()); + inputUnplayedMsMax(inputMsUnplayed.getWindowMax()); + outputUnplayedMsMax(outputMsUnplayed.getWindowMax()); + + sentTimegapMsMax(timegaps.getMax() / USECS_PER_MSEC); + sentTimegapMsAvg(timegaps.getAverage() / USECS_PER_MSEC); + sentTimegapMsMaxWindow(timegaps.getWindowMax() / USECS_PER_MSEC); + sentTimegapMsAvgWindow(timegaps.getWindowAverage() / USECS_PER_MSEC); +} + +void AudioStatsInterface::updateInjectorStreams(const QHash& stats) { + // Get existing injectors + auto injectorIds = _injectors->dynamicPropertyNames(); + + // Go over reported injectors + QHash::const_iterator injector = stats.constBegin(); + while (injector != stats.constEnd()) { + const auto id = injector.key().toByteArray(); + // Mark existing injector (those left will be removed) + injectorIds.removeOne(id); + auto injectorProperty = _injectors->property(id); + // Add new injector + if (!injectorProperty.isValid()) { + injectorProperty = QVariant::fromValue(new AudioStreamStatsInterface(this)); + _injectors->setProperty(id, injectorProperty); + } + // Update property with reported injector + injectorProperty.value()->updateStream(injector.value()); + ++injector; + } + + // Remove unreported injectors + for (auto& id : injectorIds) { + _injectors->property(id).value()->deleteLater(); + _injectors->setProperty(id, QVariant()); + } + + emit injectorStreamsChanged(); +} diff --git a/libraries/audio-client-core/src/AudioIOStats.h b/libraries/audio-client-core/src/AudioIOStats.h new file mode 100644 index 00000000000..3a996663541 --- /dev/null +++ b/libraries/audio-client-core/src/AudioIOStats.h @@ -0,0 +1,425 @@ +// +// AudioIOStats.h +// interface/src/audio +// +// Created by Stephen Birarda on 2014-12-16. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_AudioIOStats_h +#define hifi_AudioIOStats_h + +#include "MovingMinMaxAvg.h" + +#include +#include + +#include +#include +#include + +class MixedProcessedAudioStream; + +#define AUDIO_PROPERTY(TYPE, NAME) \ + Q_PROPERTY(TYPE NAME READ NAME NOTIFY NAME##Changed) \ + public: \ + TYPE NAME() const { return _##NAME; } \ + void NAME(TYPE value) { \ + if (_##NAME != value) { \ + _##NAME = value; \ + emit NAME##Changed(value); \ + } \ + } \ + Q_SIGNAL void NAME##Changed(TYPE value); \ + private: \ + TYPE _##NAME{ (TYPE)0 }; + +class AudioStreamStatsInterface : public QObject { + Q_OBJECT + + /*@jsdoc + * Statistics for an audio stream. + * + *

Provided in properties of the {@link AudioStats} API.

+ * + * @class AudioStats.AudioStreamStats + * @hideconstructor + * + * @hifi-interface + * @hifi-client-entity + * @hifi-avatar + * + * @property {number} dropCount - The number of silent or old audio frames dropped. + * Read-only. + * @property {number} framesAvailable - The number of audio frames containing data available. + * Read-only. + * @property {number} framesAvailableAvg - The time-weighted average of audio frames containing data available. + * Read-only. + * @property {number} framesDesired - The desired number of audio frames for the jitter buffer. + * Read-only. + * @property {number} lastStarveDurationCount - The most recent number of consecutive times that audio frames have not been + * available for processing. + * Read-only. + * @property {number} lossCount - The total number of audio packets lost. + * Read-only. + * @property {number} lossCountWindow - The number of audio packets lost since the previous statistic. + * Read-only. + * @property {number} lossRate - The ratio of the total number of audio packets lost to the total number of audio packets + * expected. + * Read-only. + * @property {number} lossRateWindow - The ratio of the number of audio packets lost to the number of audio packets + * expected since the previous statistic. + * Read-only. + * @property {number} overflowCount - The number of times that the audio ring buffer has overflowed. + * Read-only. + * @property {number} starveCount - The total number of times that audio frames have not been available for processing. + * Read-only. + * @property {number} timegapMsAvg - The overall average time between data packets, in ms. + * Read-only. + * @property {number} timegapMsAvgWindow - The recent average time between data packets, in ms. + * Read-only. + * @property {number} timegapMsMax - The overall maximum time between data packets, in ms. + * Read-only. + * @property {number} timegapMsMaxWindow - The recent maximum time between data packets, in ms. + * Read-only. + * @property {number} unplayedMsMax - The duration of audio waiting to be played, in ms. + * Read-only. + */ + + /*@jsdoc + * Triggered when the ratio of the total number of audio packets lost to the total number of audio packets expected changes. + * @function AudioStats.AudioStreamStats.lossRateChanged + * @param {number} lossRate - The ratio of the total number of audio packets lost to the total number of audio packets + * expected. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, lossRate) + + /*@jsdoc + * Triggered when the total number of audio packets lost changes. + * @function AudioStats.AudioStreamStats.lossCountChanged + * @param {number} lossCount - The total number of audio packets lost. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, lossCount) + + /*@jsdoc + * Triggered when the ratio of the number of audio packets lost to the number of audio packets expected since the previous + * statistic changes. + * @function AudioStats.AudioStreamStats.lossRateWindowChanged + * @param {number} lossRateWindow - The ratio of the number of audio packets lost to the number of audio packets expected + * since the previous statistic. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, lossRateWindow) + + /*@jsdoc + * Triggered when the number of audio packets lost since the previous statistic changes. + * @function AudioStats.AudioStreamStats.lossCountWindowChanged + * @param {number} lossCountWindow - The number of audio packets lost since the previous statistic. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, lossCountWindow) + + /*@jsdoc + * Triggered when the desired number of audio frames for the jitter buffer changes. + * @function AudioStats.AudioStreamStats.framesDesiredChanged + * @param {number} framesDesired - The desired number of audio frames for the jitter buffer. + * @returns {Signal} + */ + AUDIO_PROPERTY(int, framesDesired) + + /*@jsdoc + * Triggered when the number of audio frames containing data available changes. + * @function AudioStats.AudioStreamStats.framesAvailableChanged + * @param {number} framesAvailable - The number of audio frames containing data available. + * @returns {Signal} + */ + AUDIO_PROPERTY(int, framesAvailable) + + /*@jsdoc + * Triggered when the time-weighted average of audio frames containing data available changes. + * @function AudioStats.AudioStreamStats.framesAvailableAvgChanged + * @param {number} framesAvailableAvg - The time-weighted average of audio frames containing data available. + * @returns {Signal} + */ + AUDIO_PROPERTY(int, framesAvailableAvg) + + /*@jsdoc + * Triggered when the duration of audio waiting to be played changes. + * @function AudioStats.AudioStreamStats.unplayedMsMaxChanged + * @param {number} unplayedMsMax - The duration of audio waiting to be played, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, unplayedMsMax) + + /*@jsdoc + * Triggered when the total number of times that audio frames have not been available for processing changes. + * @function AudioStats.AudioStreamStats.starveCountChanged + * @param {number} starveCount - The total number of times that audio frames have not been available for processing. + * @returns {Signal} + */ + AUDIO_PROPERTY(int, starveCount) + + /*@jsdoc + * Triggered when the most recenbernumber of consecutive times that audio frames have not been available for processing + * changes. + * @function AudioStats.AudioStreamStats.lastStarveDurationCountChanged + * @param {number} lastStarveDurationCount - The most recent number of consecutive times that audio frames have not been + * available for processing. + * @returns {Signal} + */ + AUDIO_PROPERTY(int, lastStarveDurationCount) + + /*@jsdoc + * Triggered when the number of silent or old audio frames dropped changes. + * @function AudioStats.AudioStreamStats.dropCountChanged + * @param {number} dropCount - The number of silent or old audio frames dropped. + * @returns {Signal} + */ + AUDIO_PROPERTY(int, dropCount) + + /*@jsdoc + * Triggered when the number of times that the audio ring buffer has overflowed changes. + * @function AudioStats.AudioStreamStats.overflowCountChanged + * @param {number} overflowCount - The number of times that the audio ring buffer has overflowed. + * @returns {Signal} + */ + AUDIO_PROPERTY(int, overflowCount) + + /*@jsdoc + * Triggered when the overall maximum time between data packets changes. + * @function AudioStats.AudioStreamStats.timegapMsMaxChanged + * @param {number} timegapMsMax - The overall maximum time between data packets, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, timegapMsMax) + + /*@jsdoc + * Triggered when the overall average time between data packets changes. + * @function AudioStats.AudioStreamStats.timegapMsAvgChanged + * @param {number} timegapMsAvg - The overall average time between data packets, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, timegapMsAvg) + + /*@jsdoc + * Triggered when the recent maximum time between data packets changes. + * @function AudioStats.AudioStreamStats.timegapMsMaxWindowChanged + * @param {number} timegapMsMaxWindow - The recent maximum time between data packets, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, timegapMsMaxWindow) + + /*@jsdoc + * Triggered when the recent average time between data packets changes. + * @function AudioStats.AudioStreamStats.timegapMsAvgWindowChanged + * @param {number} timegapMsAvgWindow - The recent average time between data packets, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, timegapMsAvgWindow) + +public: + void updateStream(const AudioStreamStats& stats); + +private: + friend class AudioStatsInterface; + AudioStreamStatsInterface(QObject* parent); +}; + +class AudioStatsInterface : public QObject { + Q_OBJECT + + /*@jsdoc + * The AudioStats API provides statistics of the client and mixer audio. + * + * @namespace AudioStats + * + * @hifi-interface + * @hifi-client-entity + * @hifi-avatar + * + * @property {AudioStats.AudioStreamStats} clientStream - Statistics of the client's audio stream. + * Read-only. + * @property {number} inputReadMsMax - The maximum duration of a block of audio data recently read from the microphone, in + * ms. + * Read-only. + * @property {number} inputUnplayedMsMax - The maximum duration of microphone audio recently in the input buffer waiting to + * be played, in ms. + * Read-only. + * @property {AudioStats.AudioStreamStats} mixerStream - Statistics of the audio mixer's stream. + * Read-only. + * @property {number} outputUnplayedMsMax - The maximum duration of output audio recently in the output buffer waiting to + * be played, in ms. + * Read-only. + * @property {number} pingMs - The current ping time to the audio mixer, in ms. + * Read-only. + * @property {number} sentTimegapMsAvg - The overall average time between sending data packets to the audio mixer, in ms. + * Read-only. + * @property {number} sentTimegapMsAvgWindow - The recent average time between sending data packets to the audio mixer, in + * ms. + * Read-only. + * @property {number} sentTimegapMsMax - The overall maximum time between sending data packets to the audio mixer, in ms. + * Read-only. + * @property {number} sentTimegapMsMaxWindow - The recent maximum time between sending data packets to the audio mixer, in + * ms. + * Read-only. + */ + + /*@jsdoc + * Triggered when the ping time to the audio mixer changes. + * @function AudioStats.pingMsChanged + * @param {number} pingMs - The ping time to the audio mixer, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, pingMs); + + + /*@jsdoc + * Triggered when the maximum duration of a block of audio data recently read from the microphone changes. + * @function AudioStats.inputReadMsMaxChanged + * @param {number} inputReadMsMax - The maximum duration of a block of audio data recently read from the microphone, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, inputReadMsMax); + + /*@jsdoc + * Triggered when the maximum duration of microphone audio recently in the input buffer waiting to be played changes. + * @function AudioStats.inputUnplayedMsMaxChanged + * @param {number} inputUnplayedMsMax - The maximum duration of microphone audio recently in the input buffer waiting to be + * played, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, inputUnplayedMsMax); + + /*@jsdoc + * Triggered when the maximum duration of output audio recently in the output buffer waiting to be played changes. + * @function AudioStats.outputUnplayedMsMaxChanged + * @param {number} outputUnplayedMsMax - The maximum duration of output audio recently in the output buffer waiting to be + * played, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(float, outputUnplayedMsMax); + + + /*@jsdoc + * Triggered when the overall maximum time between sending data packets to the audio mixer changes. + * @function AudioStats.sentTimegapMsMaxChanged + * @param {number} sentTimegapMsMax - The overall maximum time between sending data packets to the audio mixer, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, sentTimegapMsMax); + + /*@jsdoc + * Triggered when the overall average time between sending data packets to the audio mixer changes. + * @function AudioStats.sentTimegapMsAvgChanged + * @param {number} sentTimegapMsAvg - The overall average time between sending data packets to the audio mixer, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, sentTimegapMsAvg); + + /*@jsdoc + * Triggered when the recent maximum time between sending data packets to the audio mixer changes. + * @function AudioStats.sentTimegapMsMaxWindowChanged + * @param {number} sentTimegapMsMaxWindow - The recent maximum time between sending data packets to the audio mixer, in ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, sentTimegapMsMaxWindow); + + /*@jsdoc + * Triggered when the recent average time between sending data packets to the audio mixer changes. + * @function AudioStats.sentTimegapMsAvgWindowChanged + * @param {number} sentTimegapMsAvgWindow - The recent average time between sending data packets to the audio mixer, in + * ms. + * @returns {Signal} + */ + AUDIO_PROPERTY(quint64, sentTimegapMsAvgWindow); + + Q_PROPERTY(AudioStreamStatsInterface* mixerStream READ getMixerStream NOTIFY mixerStreamChanged); + Q_PROPERTY(AudioStreamStatsInterface* clientStream READ getClientStream NOTIFY clientStreamChanged); + + // FIXME: The injectorStreams property isn't available in JavaScript but the notification signal is. + Q_PROPERTY(QObject* injectorStreams READ getInjectorStreams NOTIFY injectorStreamsChanged); + +public: + AudioStreamStatsInterface* getMixerStream() const { return _mixer; } + AudioStreamStatsInterface* getClientStream() const { return _client; } + QObject* getInjectorStreams() const { return _injectors; } + + void updateLocalBuffers(const MovingMinMaxAvg& inputMsRead, + const MovingMinMaxAvg& inputMsUnplayed, + const MovingMinMaxAvg& outputMsUnplayed, + const MovingMinMaxAvg& timegaps); + void updateMixerStream(const AudioStreamStats& stats) { _mixer->updateStream(stats); emit mixerStreamChanged(); } + void updateClientStream(const AudioStreamStats& stats) { _client->updateStream(stats); emit clientStreamChanged(); } + void updateInjectorStreams(const QHash& stats); + +signals: + + /*@jsdoc + * Triggered when the mixer's stream statistics have been updated. + * @function AudioStats.mixerStreamChanged + * @returns {Signal} + */ + void mixerStreamChanged(); + + /*@jsdoc + * Triggered when the client's stream statisticss have been updated. + * @function AudioStats.clientStreamChanged + * @returns {Signal} + */ + void clientStreamChanged(); + + /*@jsdoc + * Triggered when the injector streams' statistics have been updated. + *

Note: The injector streams' statistics are currently not provided.

+ * @function AudioStats.injectorStreamsChanged + * @returns {Signal} + */ + void injectorStreamsChanged(); + +private: + friend class AudioIOStats; + AudioStatsInterface(QObject* parent); + AudioStreamStatsInterface* _client; + AudioStreamStatsInterface* _mixer; + QObject* _injectors; +}; + +class AudioIOStats : public QObject { + Q_OBJECT +public: + AudioIOStats(MixedProcessedAudioStream* receivedAudioStream); + + void reset(); + + AudioStatsInterface* data() const { return _interface; } + + void updateInputMsRead(float ms) const { _inputMsRead.update(ms); } + void updateInputMsUnplayed(float ms) const { _inputMsUnplayed.update(ms); } + void updateOutputMsUnplayed(float ms) const { _outputMsUnplayed.update(ms); } + void sentPacket() const; + + void publish(); + +public slots: + void processStreamStatsPacket(QSharedPointer message, SharedNodePointer sendingNode); + +private: + AudioStatsInterface* _interface; + + mutable MovingMinMaxAvg _inputMsRead; + mutable MovingMinMaxAvg _inputMsUnplayed; + mutable MovingMinMaxAvg _outputMsUnplayed; + + mutable quint64 _lastSentPacketTime; + mutable MovingMinMaxAvg _packetTimegaps; + + MixedProcessedAudioStream* _receivedAudioStream; + QHash _injectorStreams; +}; + +#endif // hifi_AudioIOStats_h diff --git a/libraries/audio-client-core/src/AudioPacketHandler.h b/libraries/audio-client-core/src/AudioPacketHandler.h new file mode 100644 index 00000000000..7c6c78fe638 --- /dev/null +++ b/libraries/audio-client-core/src/AudioPacketHandler.h @@ -0,0 +1,465 @@ +// +// AudioPacketHandler.h +// libraries/audio-client-core/src +// +// Created by Nshan G. on 4 July 2022 +// Copyright 2013 High Fidelity, Inc. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef LIBRARIES_AUDIO_CLIENT_CORE_SRC_AUDIOPACKETHANDLER_H +#define LIBRARIES_AUDIO_CLIENT_CORE_SRC_AUDIOPACKETHANDLER_H + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AudioIOStats.h" + +#if defined(WEBRTC_AUDIO) +# define WEBRTC_APM_DEBUG_DUMP 0 +# include +# include "modules/audio_processing/audio_processing_impl.h" +#endif + +#ifdef _WIN32 +#pragma warning( push ) +#pragma warning( disable : 4273 ) +#pragma warning( disable : 4305 ) +#endif + +#ifdef _WIN32 +#pragma warning( pop ) +#endif + +#if defined (Q_OS_ANDROID) +#define VOICE_RECOGNITION "voicerecognition" +#define VOICE_COMMUNICATION "voicecommunication" + +#define SETTING_AEC_KEY "Android/aec" +#define DEFAULT_AEC_ENABLED true +#endif + +class QAudioInput; +class QAudioOutput; +class QIODevice; + +class Transform; +class NLPacket; + +#define DEFAULT_STARVE_DETECTION_ENABLED true +#define DEFAULT_BUFFER_FRAMES 1 + +inline auto defaultAudioPositionGetter() { + return Vectors::ZERO; +} + +inline auto defaultAudioOrientationGetter() { + return Quaternions::IDENTITY; +} + +struct AudioFormat { + enum SampleTag { + Signed16, + Float + }; + + SampleTag sampleType = SampleTag(-1); + int sampleRate = -1; + int channelCount = -1; + + int getSampleSize() const { + switch(sampleType) { + case Signed16: return sizeof(int16_t); + case Float: return sizeof(float) ; + default: return 0; + }; + }; + + int getSampleBits() const { + return getSampleSize() * CHAR_BIT; + }; + + qint32 bytesForDuration(qint64 microseconds) const { + return microseconds * sampleRate * channelCount * getSampleSize() / USECS_PER_SECOND; + } + + bool operator==(const AudioFormat& other) const { + return sampleType == other.sampleType && sampleRate == other.sampleRate && channelCount == other.channelCount; + } + bool operator!=(const AudioFormat& other) const { + return !(*this == other); + } + + bool isValid() const { + return sampleRate != -1 && channelCount != -1 && + (sampleType == Signed16 || sampleType == Float); + } +}; + +template +class AudioPacketHandler { + using LocalInjectorsStream = AudioMixRingBuffer; +public: + static const int MIN_BUFFER_FRAMES = 1; + static const int MAX_BUFFER_FRAMES = 20; + + using AudioPositionGetter = std::function; + using AudioOrientationGetter = std::function; + + using Mutex = std::mutex; + using Lock = std::unique_lock; + + class AudioOutputIODevice : public QIODevice { + public: + AudioOutputIODevice(LocalInjectorsStream& localInjectorsStream, MixedProcessedAudioStream& receivedAudioStream, + AudioPacketHandler* audio) : + _localInjectorsStream(localInjectorsStream), _receivedAudioStream(receivedAudioStream), + _audio(audio), _unfulfilledReads(0) {} + + void open() { QIODevice::open(QIODevice::ReadOnly | QIODevice::Unbuffered); } + qint64 readData(char* data, qint64 maxSize) override; + qint64 writeData(const char* data, qint64 maxSize) override { return 0; } + int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; } + private: + using QIODevice::open; // silence overloaded virtual warning + + LocalInjectorsStream& _localInjectorsStream; + MixedProcessedAudioStream& _receivedAudioStream; + AudioPacketHandler* _audio; + int _unfulfilledReads; + }; + + void startThread(); + void negotiateAudioFormat(); + void selectAudioFormat(const QString& selectedCodecName); + + Q_INVOKABLE QString getSelectedAudioFormat() const { return _selectedCodecName; } + Q_INVOKABLE bool getNoiseGateOpen() const { return _audioGateOpen; } + Q_INVOKABLE float getSilentInboundPPS() const { return _silentInbound.rate(); } + Q_INVOKABLE float getAudioInboundPPS() const { return _audioInbound.rate(); } + Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); } + Q_INVOKABLE float getAudioOutboundPPS() const { return _audioOutbound.rate(); } + + const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; } + MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; } + + float getLastInputLoudness() const { return _lastInputLoudness; } + + float getTimeSinceLastClip() const { return _timeSinceLastClip; } + float getAudioAverageInputLoudness() const { return _lastInputLoudness; } + + const AudioIOStats& getStats() const { return _stats; } + + bool isSimulatingJitter() { return _gate.isSimulatingJitter(); } + void setIsSimulatingJitter(bool enable) { _gate.setIsSimulatingJitter(enable); } + + int getGateThreshold() { return _gate.getThreshold(); } + void setGateThreshold(int threshold) { _gate.setThreshold(threshold); } + + void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; } + void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; } + + void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; } + + Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale); + + // FIXME: CRTP should override virtual member of in audio-client/sec/AudioClient + bool outputLocalInjector(const AudioInjectorPointer& injector); + + // HifiAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const; + // QList getAudioDevices(QAudio::Mode mode) const; + + // void enablePeakValues(bool enable) { _enablePeakValues = enable; } + // bool peakValuesAvailable() const; + + // bool getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName); + + void setAudioPaused(bool pause); + + // FIXME: CRTP should override virtual member of in audio-client/sec/AudioClient + AudioSolo& getAudioSolo() { return _solo; } + + int getNumLocalInjectors(); + + void start(); + void stop(); + + void handleAudioEnvironmentDataPacket(QSharedPointer message); + void handleAudioDataPacket(QSharedPointer message); + void handleNoisyMutePacket(QSharedPointer message); + void handleMuteEnvironmentPacket(QSharedPointer message); + void handleSelectedAudioFormat(QSharedPointer message); + void handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec); + + void sendDownstreamAudioStatsPacket() { _stats.publish(); } + void handleMicAudioInput(const char* data, int size); + void sendInput(); + void audioInputStateChanged(QAudio::State state); + void handleRecordedAudioInput(const QByteArray& audio); + void reset(); + void audioMixerKilled(); + + void setMuted(bool muted, bool emitSignal = true); + bool isMuted() { return _isMuted; } + + void setNoiseReduction(bool isNoiseGateEnabled, bool emitSignal = true); + bool isNoiseReductionEnabled() const { return _isNoiseGateEnabled; } + + void setNoiseReductionAutomatic(bool isNoiseGateAutomatic, bool emitSignal = true); + bool isNoiseReductionAutomatic() const { return _isNoiseReductionAutomatic; } + + void setNoiseReductionThreshold(float noiseReductionThreshold, bool emitSignal = true); + float noiseReductionThreshold() const { return _noiseReductionThreshold; } + + void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true); + bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; } + + // FIXME: CRTP these should override virtual members of in audio-client/sec/AudioClient + bool getServerEcho() { return _shouldEchoToServer; } + void setServerEcho(bool serverEcho) { _shouldEchoToServer = serverEcho; } + void toggleServerEcho() { _shouldEchoToServer = !_shouldEchoToServer; } + + void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer); + void sendMuteEnvironmentPacket(); + + int setOutputBufferSize(int numFrames, bool persist = true); + + void setReverb(bool reverb); + void setReverbOptions(const AudioEffectOptions* options); + + void setLocalInjectorGain(float gain) { _localInjectorGain = gain; }; + void setSystemInjectorGain(float gain) { _systemInjectorGain = gain; }; + void setOutputGain(float gain) { _outputGain = gain; }; + +protected: + AudioPacketHandler(); + ~AudioPacketHandler(); + + void cleanupInput(); + bool setupInput(AudioFormat); + void setupDummyInput(); + bool isDummyInput(); + + void cleanupOutput(); + bool setupOutput(AudioFormat); + + AudioOutputIODevice _audioOutputIODevice {_localInjectorsStream, _receivedAudioStream, this}; + bool _isMuted {false}; + glm::vec3 avatarBoundingBoxCorner {}; + glm::vec3 avatarBoundingBoxScale {}; + AudioFormat _inputFormat {}; + AudioFormat _outputFormat {}; + QString _selectedCodecName; + +private: + static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES{ 100 }; + // OUTPUT_CHANNEL_COUNT is audio pipeline output format, which is always 2 channel. + // _outputFormat.channelCount() is device output format, which may be 1 or multichannel. + static const int OUTPUT_CHANNEL_COUNT{ 2 }; + static const int STARVE_DETECTION_THRESHOLD{ 3 }; + static const int STARVE_DETECTION_PERIOD{ 10 * 1000 }; // 10 Seconds + + friend class CheckDevicesThread; + friend class LocalInjectorsThread; + + Derived& derived(); + const Derived& derived() const; + + float loudnessToLevel(float loudness); + + void outputFormatChanged(); + void handleAudioInput(QByteArray& audioBuffer); + void prepareLocalAudioInjectors(std::unique_ptr localAudioLock = nullptr); + bool mixLocalAudioInjectors(float* mixBuffer); + float azimuthForSource(const glm::vec3& relativePosition); + float gainForSource(float distance, float volume); + + class Gate { + public: + Gate(AudioPacketHandler* audioClient); + + bool isSimulatingJitter() { return _isSimulatingJitter; } + void setIsSimulatingJitter(bool enable); + + int getThreshold() { return _threshold; } + void setThreshold(int threshold); + + void insert(QSharedPointer message); + + private: + void flush(); + + AudioPacketHandler* _audioClient; + std::queue> _queue; + std::mutex _mutex; + + int _index {0}; + int _threshold {1}; + bool _isSimulatingJitter {false}; + }; + + Gate _gate{ this }; + + Mutex _injectorsMutex; + QTimer* _dummyAudioInput {nullptr}; + AudioFormat _desiredInputFormat; + std::atomic _audioOutputInitialized {false}; + AudioFormat _desiredOutputFormat; + int _outputFrameSize {0}; + int _numOutputCallbackBytes {0}; + std::vector _inputTypeConversionBuffer {}; + AudioRingBuffer _inputRingBuffer {0}; + LocalInjectorsStream _localInjectorsStream {0 , 1}; + // In order to use _localInjectorsStream as a lock-free pipe, + // use it with a single producer/consumer, and track available samples and injectors + std::atomic _localSamplesAvailable {0}; + std::atomic _localInjectorsAvailable {false}; + MixedProcessedAudioStream _receivedAudioStream {RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES}; + + quint64 _outputStarveDetectionStartTimeMsec {0}; + int _outputStarveDetectionCount {0}; + + int _sessionOutputBufferSizeFrames {DEFAULT_BUFFER_FRAMES}; + + float _lastRawInputLoudness {0.0f}; // before mute/gate + float _lastSmoothedRawInputLoudness {0.0f}; + float _lastInputLoudness {0.0f}; // after mute/gate + float _timeSinceLastClip {-1.0f}; + int _totalInputAudioSamples; + + bool _shouldEchoLocally {false}; + bool _shouldEchoToServer {false}; + bool _isNoiseGateEnabled {true}; + bool _isNoiseReductionAutomatic {true}; + float _noiseReductionThreshold {0.1f}; + bool _isAECEnabled {true}; + + bool _reverb {false}; + AudioEffectOptions _scriptReverbOptions; + AudioEffectOptions _zoneReverbOptions; + AudioEffectOptions* _reverbOptions {&_scriptReverbOptions}; + AudioReverb _sourceReverb {AudioConstants::SAMPLE_RATE}; + AudioReverb _listenerReverb {AudioConstants::SAMPLE_RATE}; + AudioReverb _localReverb {AudioConstants::SAMPLE_RATE}; + + // possible streams needed for resample + AudioSRC* _inputToNetworkResampler {nullptr}; + AudioSRC* _networkToOutputResampler {nullptr}; + AudioSRC* _localToOutputResampler {nullptr}; + + // for network audio (used by network audio thread) + int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + + // for output audio (used by this thread) + int _outputPeriod {0}; + float* _outputMixBuffer {NULL}; + int16_t* _outputScratchBuffer {NULL}; + std::atomic _outputGain {1.0f}; + float _lastOutputGain {1.0f}; + + // for local audio (used by audio injectors thread) + std::atomic _localInjectorGain {1.0f}; + std::atomic _systemInjectorGain {1.0f}; + float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC]; + float* _localOutputMixBuffer {NULL}; + Mutex _localAudioMutex; + AudioLimiter _audioLimiter {AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT}; + + // Adds Reverb + void configureReverb(); + void updateReverbOptions(); + +#if defined(WEBRTC_AUDIO) + static const int WEBRTC_SAMPLE_RATE_MAX = 96000; + static const int WEBRTC_CHANNELS_MAX = 2; + static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000; + + webrtc::AudioProcessing* _apm {nullptr}; + + int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {}; + int _numFifoFarEnd = 0; // numFrames saved in fifo + + void configureWebrtc(); + void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate); + void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate); +#endif + + // Callback acceleration dependent calculations + int calculateNumberOfInputCallbackBytes(const AudioFormat& format) const; + int calculateNumberOfFrameSamples(int numBytes) const; + + quint16 _outgoingAvatarAudioSequenceNumber {0}; + + AudioIOStats _stats {&_receivedAudioStream}; + + AudioGate* _audioGate {nullptr}; + bool _audioGateOpen {true}; + + AudioPositionGetter _positionGetter{ defaultAudioPositionGetter }; + AudioOrientationGetter _orientationGetter{ defaultAudioOrientationGetter }; + + bool _hasReceivedFirstPacket {false}; + + QVector _activeLocalAudioInjectors; + + bool _isPlayingBackRecording {false}; + bool _audioPaused {false}; + + std::shared_ptr _codec; + Encoder* _encoder {nullptr}; // for outbound mic stream + + RateCounter<> _silentOutbound; + RateCounter<> _audioOutbound; + RateCounter<> _silentInbound; + RateCounter<> _audioInbound; + +#if defined(Q_OS_ANDROID) + bool _shouldRestartInputSetup {true}; // Should we restart the input device because of an unintended stop? +#endif + + AudioSolo _solo; + + QFuture _localPrepInjectorFuture; + + bool _isRecording {false}; +}; + +#endif /* end of include guard */ diff --git a/libraries/audio-client-core/src/AudioPacketHandler.hpp b/libraries/audio-client-core/src/AudioPacketHandler.hpp new file mode 100644 index 00000000000..96c5d74961a --- /dev/null +++ b/libraries/audio-client-core/src/AudioPacketHandler.hpp @@ -0,0 +1,2035 @@ +// +// AudioPacketHandler.hpp +// libraries/audio-client-core/src +// +// Created by Nshan G on 4 July 2022. +// Copyright 2013 High Fidelity, Inc. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef LIBRARIES_AUDIO_CLIENT_CORE_SRC_AUDIOPACKETHANDLER_HPP +#define LIBRARIES_AUDIO_CLIENT_CORE_SRC_AUDIOPACKETHANDLER_HPP + +#include "AudioPacketHandler.h" + +#include +#include +#include + +#include +#include +#include + +#ifdef __APPLE__ +#include +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN 1 +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "AudioClientLogging.h" +#include "AudioLogging.h" +#include "AudioHelpers.h" + +using Lock = std::unique_lock; + +inline QDebug& operator<<(QDebug& debug, const AudioFormat& audioFormat) { + QString sampleTypeString = audioFormat.sampleType == AudioFormat::Signed16 ? "Signed 16bit" : + audioFormat.sampleType == AudioFormat::Float ? "Float" : + "!INVALID!"; + return debug << QString("AudioFormat{ sampleType: %1, sampleRate: %2, channelCount: %3 }") + .arg(sampleTypeString, QString::number(audioFormat.sampleRate), QString::number(audioFormat.channelCount)); +} + +template +auto channelUpmix(const Source* source, Destination* destination, int numSamples, int numExtraChannels) { + Source* dest = reinterpret_cast(destination); + for (int i = 0; i < numSamples/2; i++) { + + *dest++ = *source++; // left + *dest++ = *source++; // right + // and the rest + for (int n = 0; n < numExtraChannels; n++) { + *dest++ = 0; + } + } + return reinterpret_cast(dest) - reinterpret_cast(destination); +} + +template +auto channelDownmix(const Source* source, Destination* destination, int numSamples) { + Source* dest = reinterpret_cast(destination); + for (int i = 0; i < numSamples/2; i++) { + + // read 2 samples + int16_t left = *source++; + int16_t right = *source++; + + // write 1 sample + *dest++ = (left + right) / 2; + } + return reinterpret_cast(dest) - reinterpret_cast(destination); +} + +template +auto copyWithChannelConversion(const Source* source, int sourceChannels, Destination* destination, int destinationChannels, int numSamples) { + if (destinationChannels == sourceChannels) { + const std::ptrdiff_t byteCount = numSamples * sizeof(Source); + memcpy(destination, source, byteCount); + return byteCount; + } else if (destinationChannels > sourceChannels) { + int extraChannels = destinationChannels - sourceChannels; + return channelUpmix(source, destination, numSamples, extraChannels); + } else { + return channelDownmix(source, destination, numSamples); + } +} + +inline bool detectClipping(int16_t* samples, int numSamples, int numChannels) { + + const int32_t CLIPPING_THRESHOLD = 32392; // -0.1 dBFS + const int CLIPPING_DETECTION = 3; // consecutive samples over threshold + + bool isClipping = false; + + if (numChannels == 2) { + int oversLeft = 0; + int oversRight = 0; + + for (int i = 0; i < numSamples/2; i++) { + int32_t left = std::abs((int32_t)samples[2*i+0]); + int32_t right = std::abs((int32_t)samples[2*i+1]); + + if (left > CLIPPING_THRESHOLD) { + isClipping |= (++oversLeft >= CLIPPING_DETECTION); + } else { + oversLeft = 0; + } + if (right > CLIPPING_THRESHOLD) { + isClipping |= (++oversRight >= CLIPPING_DETECTION); + } else { + oversRight = 0; + } + } + } else { + int overs = 0; + + for (int i = 0; i < numSamples; i++) { + int32_t sample = std::abs((int32_t)samples[i]); + + if (sample > CLIPPING_THRESHOLD) { + isClipping |= (++overs >= CLIPPING_DETECTION); + } else { + overs = 0; + } + } + } + + return isClipping; +} + +inline float computeLoudness(int16_t* samples, int numSamples) { + + float scale = numSamples ? 1.0f / numSamples : 0.0f; + + int32_t loudness = 0; + for (int i = 0; i < numSamples; i++) { + loudness += std::abs((int32_t)samples[i]); + } + return (float)loudness * scale; +} + +template +static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float gain1) { + + // fast path for unity gain + if (gain0 == 1.0f && gain1 == 1.0f) { + return; + } + + // cubic poly from gain0 to gain1 + float c3 = -2.0f * (gain1 - gain0); + float c2 = 3.0f * (gain1 - gain0); + float c0 = gain0; + + float t = 0.0f; + float tStep = 1.0f / numFrames; + + for (int i = 0; i < numFrames; i++) { + + // evaluate poly over t=[0,1) + float gain = (c3 * t + c2) * t * t + c0; + t += tStep; + + // apply gain to all channels + for (int ch = 0; ch < NUM_CHANNELS; ch++) { + buffer[NUM_CHANNELS*i + ch] *= gain; + } + } +} + +inline float convertToFloat(int16_t sample) { + return static_cast(sample * (1.f / std::numeric_limits::max())); +} + +inline float convertToSignedInt(float sample) { + return static_cast(sample * std::numeric_limits::max()); +} + +template +Derived& AudioPacketHandler::derived() { + return *static_cast(this); +} + +template +const Derived& AudioPacketHandler::derived() const { + return *static_cast(this); +} + +// FIXME: CRTP this will be needed in audio-client/src/AudioClient +// QAudioFormat qAudioFormatFrom(AudioFormat format) { +// QAudioFormat result; +// result.setSampleRate(format.sampleRate); +// result.setSampleSize(format.getSampleBits()); +// result.setCodec("audio/pcm"); +// result.setSampleType(format.sampleType == AudioFormat::Signed16 ? QAudioFormat::SignedInt : +// format.sampleType == AudioFormat::Float ? QAudioFormat::Float : +// QAudioFormat::Unknown); +// result.setByteOrder(QAudioFormat::LittleEndian); +// result.setChannelCount(format.channelCount); +// return result; +// } + +template +AudioPacketHandler::AudioPacketHandler() { + + // FIXME: CRTP, + // _sessionOutputBufferSizeFrames = _outputBufferSizeFrames.get(); + + // set up the desired audio format + _desiredInputFormat.sampleRate = AudioConstants::SAMPLE_RATE; + _desiredInputFormat.sampleType = AudioFormat::Signed16; + _desiredInputFormat.channelCount = AudioConstants::MONO; + + _desiredOutputFormat = _desiredInputFormat; + _desiredOutputFormat.channelCount = OUTPUT_CHANNEL_COUNT; + + + // avoid putting a lock in the device callback + assert(_localSamplesAvailable.is_lock_free()); +} + +template +AudioPacketHandler::~AudioPacketHandler() { + + stop(); + + if (_codec && _encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + } + + if (_dummyAudioInput) { + _dummyAudioInput->stop(); + _dummyAudioInput->deleteLater(); + _dummyAudioInput = nullptr; + } +} + +template +void AudioPacketHandler::handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& receivedCodec) { + qCDebug(audioclient) << __FUNCTION__ << "sendingNode:" << *node << "currentCodec:" << currentCodec << "receivedCodec:" << receivedCodec; + selectAudioFormat(receivedCodec); +} + +template +void AudioPacketHandler::reset() { + _receivedAudioStream.reset(); + _stats.reset(); + _sourceReverb.reset(); + _listenerReverb.reset(); + _localReverb.reset(); +} + +template +void AudioPacketHandler::audioMixerKilled() { + _hasReceivedFirstPacket = false; + _outgoingAvatarAudioSequenceNumber = 0; + _stats.reset(); + // FIXME: CRTP + // emit disconnected(); +} + +template +void AudioPacketHandler::setAudioPaused(bool pause) { + if (_audioPaused != pause) { + _audioPaused = pause; + + if (!_audioPaused) { + negotiateAudioFormat(); + } + } +} + +bool sampleChannelConversion(const int16_t* sourceSamples, int16_t* destinationSamples, int numSourceSamples, + const int sourceChannelCount, const int destinationChannelCount) { + if (sourceChannelCount == 2 && destinationChannelCount == 1) { + // loop through the stereo input audio samples and average every two samples + for (int i = 0; i < numSourceSamples; i += 2) { + destinationSamples[i / 2] = (sourceSamples[i] / 2) + (sourceSamples[i + 1] / 2); + } + + return true; + } else if (sourceChannelCount == 1 && destinationChannelCount == 2) { + + // loop through the mono input audio and repeat each sample twice + for (int i = 0; i < numSourceSamples; ++i) { + destinationSamples[i * 2] = destinationSamples[(i * 2) + 1] = sourceSamples[i]; + } + + return true; + } + + return false; +} + +int possibleResampling(AudioSRC* resampler, + const int16_t* sourceSamples, int16_t* destinationSamples, + int numSourceSamples, int maxDestinationSamples, + const int sourceChannelCount, const int destinationChannelCount) { + + int numSourceFrames = numSourceSamples / sourceChannelCount; + int numDestinationFrames = 0; + + if (numSourceSamples > 0) { + if (!resampler) { + if (!sampleChannelConversion(sourceSamples, destinationSamples, numSourceSamples, + sourceChannelCount, destinationChannelCount)) { + // no conversion, we can copy the samples directly across + memcpy(destinationSamples, sourceSamples, numSourceSamples * AudioConstants::SAMPLE_SIZE); + } + numDestinationFrames = numSourceFrames; + } else { + if (sourceChannelCount != destinationChannelCount) { + + int16_t* channelConversionSamples = new int16_t[numSourceFrames * destinationChannelCount]; + + sampleChannelConversion(sourceSamples, channelConversionSamples, numSourceSamples, + sourceChannelCount, destinationChannelCount); + + numDestinationFrames = resampler->render(channelConversionSamples, destinationSamples, numSourceFrames); + + delete[] channelConversionSamples; + } else { + numDestinationFrames = resampler->render(sourceSamples, destinationSamples, numSourceFrames); + } + } + } + + int numDestinationSamples = numDestinationFrames * destinationChannelCount; + if (numDestinationSamples > maxDestinationSamples) { + qCWarning(audioclient) << "Resampler overflow! numDestinationSamples =" << numDestinationSamples + << "but maxDestinationSamples =" << maxDestinationSamples; + } + return numDestinationSamples; +} + +template +void AudioPacketHandler::start() { + // deprecate legacy settings + { + Setting::Handle::Deprecated("maxFramesOverDesired", InboundAudioStream::MAX_FRAMES_OVER_DESIRED); + Setting::Handle::Deprecated("windowStarveThreshold", InboundAudioStream::WINDOW_STARVE_THRESHOLD); + Setting::Handle::Deprecated("windowSecondsForDesiredCalcOnTooManyStarves", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_CALC_ON_TOO_MANY_STARVES); + Setting::Handle::Deprecated("windowSecondsForDesiredReduction", InboundAudioStream::WINDOW_SECONDS_FOR_DESIRED_REDUCTION); + Setting::Handle::Deprecated("useStDevForJitterCalc", InboundAudioStream::USE_STDEV_FOR_JITTER); + Setting::Handle::Deprecated("repetitionWithFade", InboundAudioStream::REPETITION_WITH_FADE); + } + + derived().connect(&_receivedAudioStream, &MixedProcessedAudioStream::processSamples, + &derived(), [this](const QByteArray& decodedBuffer, QByteArray& outputBuffer) + { processReceivedSamples(decodedBuffer, outputBuffer); }, + Qt::DirectConnection); + + // FIXME: CRTP + // connect(this, &AudioClient::changeDevice, this, [=](const HifiAudioDeviceInfo& outputDeviceInfo) { + // qCDebug(audioclient)<< "got AudioClient::changeDevice signal, about to call switchOutputToAudioDevice() outputDeviceInfo: ["<< outputDeviceInfo.deviceName() << "]"; + // switchOutputToAudioDevice(outputDeviceInfo); + // }); + + derived().connect(&_receivedAudioStream, &InboundAudioStream::mismatchedAudioCodec, + &derived(), [this](SharedNodePointer node, const QString& currentCodec, const QString& receivedCodec) + { handleMismatchAudioFormat(node, currentCodec, receivedCodec); }); + + // FIXME: CRTP + // initialize wasapi; if getAvailableDevices is called from the CheckDevicesThread before this, it will crash + // defaultAudioDeviceName(QAudio::AudioInput); + // defaultAudioDeviceName(QAudio::AudioOutput); + + // FIXME: CRTP + // start a thread to detect any device changes + // _checkDevicesTimer = new QTimer(this); + // const unsigned long DEVICE_CHECK_INTERVAL_MSECS = 2 * 1000; + // connect(_checkDevicesTimer, &QTimer::timeout, this, [=] { + // QtConcurrent::run(QThreadPool::globalInstance(), [=] { + // checkDevices(); + // // On some systems (Ubuntu) checking all the audio devices can take more than 2 seconds. To + // // avoid consuming all of the thread pool, don't start the check interval until the previous + // // check has completed. + // QMetaObject::invokeMethod(_checkDevicesTimer, "start", Q_ARG(int, DEVICE_CHECK_INTERVAL_MSECS)); + // }); + // }); + // _checkDevicesTimer->setSingleShot(true); + // _checkDevicesTimer->start(DEVICE_CHECK_INTERVAL_MSECS); + + // FIXME: CRTP + // start a thread to detect peak value changes + // _checkPeakValuesTimer = new QTimer(this); + // connect(_checkPeakValuesTimer, &QTimer::timeout, this, [this] { + // QtConcurrent::run(QThreadPool::globalInstance(), [this] { checkPeakValues(); }); + // }); + // const unsigned long PEAK_VALUES_CHECK_INTERVAL_MSECS = 50; + // _checkPeakValuesTimer->start(PEAK_VALUES_CHECK_INTERVAL_MSECS); + // + // configureReverb(); + +#if defined(WEBRTC_AUDIO) + configureWebrtc(); +#endif + + auto nodeList = DependencyManager::get(); + auto& packetReceiver = nodeList->getPacketReceiver(); + packetReceiver.registerListener(PacketType::AudioStreamStats, + PacketReceiver::makeSourcedListenerReference(&_stats, &AudioIOStats::processStreamStatsPacket)); + packetReceiver.registerListener(PacketType::AudioEnvironment, + PacketReceiver::makeUnsourcedListenerReference(&derived(), &AudioPacketHandler::handleAudioEnvironmentDataPacket)); + packetReceiver.registerListener(PacketType::SilentAudioFrame, + PacketReceiver::makeUnsourcedListenerReference(&derived(), &AudioPacketHandler::handleAudioDataPacket)); + packetReceiver.registerListener(PacketType::MixedAudio, + PacketReceiver::makeUnsourcedListenerReference(&derived(), &AudioPacketHandler::handleAudioDataPacket)); + packetReceiver.registerListener(PacketType::NoisyMute, + PacketReceiver::makeUnsourcedListenerReference(&derived(), &AudioPacketHandler::handleNoisyMutePacket)); + packetReceiver.registerListener(PacketType::MuteEnvironment, + PacketReceiver::makeUnsourcedListenerReference(&derived(), &AudioPacketHandler::handleMuteEnvironmentPacket)); + packetReceiver.registerListener(PacketType::SelectedAudioFormat, + PacketReceiver::makeUnsourcedListenerReference(&derived(), &AudioPacketHandler::handleSelectedAudioFormat)); + + auto& domainHandler = nodeList->getDomainHandler(); + derived().connect(&domainHandler, &DomainHandler::disconnectedFromDomain, &derived(), [this] { + _solo.reset(); + }); + derived().connect(nodeList.data(), &NodeList::nodeActivated, &derived(), [this](SharedNodePointer node) { + if (node->getType() == NodeType::AudioMixer) { + _solo.resend(); + negotiateAudioFormat(); + } + }); + + //initialize input to the dummy device to prevent starves + cleanupInput(); + setupDummyInput(); + + derived().onStart(); + +// FIXME: CRTP +// switchOutputToAudioDevice(defaultAudioDeviceForMode(QAudio::AudioOutput, QString())); +// #if defined(Q_OS_ANDROID) +// connect(&_checkInputTimer, &QTimer::timeout, this, &AudioClient::checkInputTimeout); +// _checkInputTimer.start(CHECK_INPUT_READS_MSECS); +// #endif +} + +template +void AudioPacketHandler::stop() { + qCDebug(audioclient) << "AudioPacketHandler::stop(), requesting audio input device to shut down"; + cleanupInput(); + qCDebug(audioclient) << "The audio input device has shut down."; + + + qCDebug(audioclient) << "AudioPacketHandler::stop(), requesting audio output device to shut down"; + cleanupOutput(); + qCDebug(audioclient) << "The audio output device has shut down."; + + // FIXME: CRTP + // Stop triggering the checks + // QObject::disconnect(_checkPeakValuesTimer, &QTimer::timeout, nullptr, nullptr); + // QObject::disconnect(_checkDevicesTimer, &QTimer::timeout, nullptr, nullptr); + // Destruction of the pointers will occur when the parent object (this) is destroyed) + // { + // Lock lock(_checkDevicesMutex); + // _checkDevicesTimer->stop(); + // _checkDevicesTimer = nullptr; + // } + // { + // Lock lock(_checkPeakValuesMutex); + // _checkPeakValuesTimer = nullptr; + // } + +// FIXME: CRTP +// #if defined(Q_OS_ANDROID) +// _checkInputTimer.stop(); +// disconnect(&_checkInputTimer, &QTimer::timeout, 0, 0); +// #endif +} + +template +void AudioPacketHandler::handleAudioEnvironmentDataPacket(QSharedPointer message) { + char bitset; + message->readPrimitive(&bitset); + + bool hasReverb = oneAtBit(bitset, HAS_REVERB_BIT); + + if (hasReverb) { + float reverbTime, wetLevel; + message->readPrimitive(&reverbTime); + message->readPrimitive(&wetLevel); + _receivedAudioStream.setReverb(reverbTime, wetLevel); + } else { + _receivedAudioStream.clearReverb(); + } +} + +template +void AudioPacketHandler::handleAudioDataPacket(QSharedPointer message) { + if (message->getType() == PacketType::SilentAudioFrame) { + _silentInbound.increment(); + } else { + _audioInbound.increment(); + } + + auto nodeList = DependencyManager::get(); + nodeList->flagTimeForConnectionStep(LimitedNodeList::ConnectionStep::ReceiveFirstAudioPacket); + + if (_audioOutputInitialized.load(std::memory_order_acquire)) { + + if (!_hasReceivedFirstPacket) { + _hasReceivedFirstPacket = true; + + // have the audio scripting interface emit a signal to say we just connected to mixer + // FIXME: CRTP + // emit receivedFirstPacket(); + } + +#if DEV_BUILD || PR_BUILD + _gate.insert(message); +#else + // Audio output must exist and be correctly set up if we're going to process received audio + _receivedAudioStream.parseData(*message); +#endif + } +} + +template +AudioPacketHandler::Gate::Gate(AudioPacketHandler* audioClient) : + _audioClient(audioClient) {} + +template +void AudioPacketHandler::Gate::setIsSimulatingJitter(bool enable) { + std::lock_guard lock(_mutex); + flush(); + _isSimulatingJitter = enable; +} + +template +void AudioPacketHandler::Gate::setThreshold(int threshold) { + std::lock_guard lock(_mutex); + flush(); + _threshold = std::max(threshold, 1); +} + +template +void AudioPacketHandler::Gate::insert(QSharedPointer message) { + std::lock_guard lock(_mutex); + + // Short-circuit for normal behavior + if (_threshold == 1 && !_isSimulatingJitter) { + _audioClient->_receivedAudioStream.parseData(*message); + return; + } + + // Throttle the current packet until the next flush + _queue.push(message); + _index++; + + // When appropriate, flush all held packets to the received audio stream + if (_isSimulatingJitter) { + // The JITTER_FLUSH_CHANCE defines the discrete probability density function of jitter (ms), + // where f(t) = pow(1 - JITTER_FLUSH_CHANCE, (t / 10) * JITTER_FLUSH_CHANCE + // for t (ms) = 10, 20, ... (because typical packet timegap is 10ms), + // because there is a JITTER_FLUSH_CHANCE of any packet instigating a flush of all held packets. + static const float JITTER_FLUSH_CHANCE = 0.6f; + // It is set at 0.6 to give a low chance of spikes (>30ms, 2.56%) so that they are obvious, + // but settled within the measured 5s window in audio network stats. + if (randFloat() < JITTER_FLUSH_CHANCE) { + flush(); + } + } else if (!(_index % _threshold)) { + flush(); + } +} + +template +void AudioPacketHandler::Gate::flush() { + // Send all held packets to the received audio stream to be (eventually) played + while (!_queue.empty()) { + _audioClient->_receivedAudioStream.parseData(*_queue.front()); + _queue.pop(); + } + _index = 0; +} + + +template +void AudioPacketHandler::handleNoisyMutePacket(QSharedPointer message) { + if (!_isMuted) { + setMuted(true); + + derived().onMutedByMixer(); + // FIXME: CRTP + // have the audio scripting interface emit a signal to say we were muted by the mixer + // emit mutedByMixer(); + } +} + +template +void AudioPacketHandler::handleMuteEnvironmentPacket(QSharedPointer message) { + glm::vec3 position; + float radius; + + message->readPrimitive(&position); + message->readPrimitive(&radius); + + // FIXME: CRTP + // emit muteEnvironmentRequested(position, radius); +} + +template +void AudioPacketHandler::negotiateAudioFormat() { + auto nodeList = DependencyManager::get(); + auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); + const auto& codecs = derived().getSupportedCodecs(); + // FIXME: CRTP + // const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + quint8 numberOfCodecs = (quint8)codecs.size(); + negotiateFormatPacket->writePrimitive(numberOfCodecs); + for (const auto& codec : codecs) { + auto codecName = codec->getName(); + negotiateFormatPacket->writeString(codecName); + } + + // grab our audio mixer from the NodeList, if it exists + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this mute packet + nodeList->sendPacket(std::move(negotiateFormatPacket), *audioMixer); + } +} + +template +void AudioPacketHandler::handleSelectedAudioFormat(QSharedPointer message) { + QString selectedCodecName = message->readString(); + selectAudioFormat(selectedCodecName); +} + +template +void AudioPacketHandler::selectAudioFormat(const QString& selectedCodecName) { + + _selectedCodecName = selectedCodecName; + + qCDebug(audioclient) << "Selected codec:" << _selectedCodecName << "; Is stereo input:" << (_desiredInputFormat.channelCount == AudioConstants::STEREO); + + // release any old codec encoder/decoder first... + if (_codec && _encoder) { + _codec->releaseEncoder(_encoder); + _encoder = nullptr; + _codec = nullptr; + } + _receivedAudioStream.cleanupCodec(); + + const auto& codecs = derived().getSupportedCodecs(); + // FIXME: CRTP + // const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (const auto& codec : codecs) { + if (_selectedCodecName == codec->getName()) { + _codec = codec; + _receivedAudioStream.setupCodec(codec, _selectedCodecName, AudioConstants::STEREO); + _encoder = codec->createEncoder(_desiredInputFormat.sampleRate, _desiredInputFormat.channelCount); + qCDebug(audioclient) << "Selected codec:" << _codec.get(); + break; + } + } + +} + +template +void AudioPacketHandler::configureReverb() { + ReverbParameters p; + + p.sampleRate = AudioConstants::SAMPLE_RATE; + p.bandwidth = _reverbOptions->getBandwidth(); + p.preDelay = _reverbOptions->getPreDelay(); + p.lateDelay = _reverbOptions->getLateDelay(); + p.reverbTime = _reverbOptions->getReverbTime(); + p.earlyDiffusion = _reverbOptions->getEarlyDiffusion(); + p.lateDiffusion = _reverbOptions->getLateDiffusion(); + p.roomSize = _reverbOptions->getRoomSize(); + p.density = _reverbOptions->getDensity(); + p.bassMult = _reverbOptions->getBassMult(); + p.bassFreq = _reverbOptions->getBassFreq(); + p.highGain = _reverbOptions->getHighGain(); + p.highFreq = _reverbOptions->getHighFreq(); + p.modRate = _reverbOptions->getModRate(); + p.modDepth = _reverbOptions->getModDepth(); + p.earlyGain = _reverbOptions->getEarlyGain(); + p.lateGain = _reverbOptions->getLateGain(); + p.earlyMixLeft = _reverbOptions->getEarlyMixLeft(); + p.earlyMixRight = _reverbOptions->getEarlyMixRight(); + p.lateMixLeft = _reverbOptions->getLateMixLeft(); + p.lateMixRight = _reverbOptions->getLateMixRight(); + p.wetDryMix = _reverbOptions->getWetDryMix(); + + _listenerReverb.setParameters(&p); + _localReverb.setParameters(&p); + + // used only for adding self-reverb to loopback audio + p.sampleRate = _outputFormat.sampleRate; + p.wetDryMix = 100.0f; + p.preDelay = 0.0f; + p.earlyGain = -96.0f; // disable ER + p.lateGain += _reverbOptions->getWetDryMix() * (24.0f / 100.0f) - 24.0f; // -0dB to -24dB, based on wetDryMix + p.lateMixLeft = 0.0f; + p.lateMixRight = 0.0f; + + _sourceReverb.setParameters(&p); +} + +template +void AudioPacketHandler::updateReverbOptions() { + bool reverbChanged = false; + if (_receivedAudioStream.hasReverb()) { + + if (_zoneReverbOptions.getReverbTime() != _receivedAudioStream.getRevebTime()) { + _zoneReverbOptions.setReverbTime(_receivedAudioStream.getRevebTime()); + reverbChanged = true; + } + if (_zoneReverbOptions.getWetDryMix() != _receivedAudioStream.getWetLevel()) { + _zoneReverbOptions.setWetDryMix(_receivedAudioStream.getWetLevel()); + reverbChanged = true; + } + + if (_reverbOptions != &_zoneReverbOptions) { + _reverbOptions = &_zoneReverbOptions; + reverbChanged = true; + } + } else if (_reverbOptions != &_scriptReverbOptions) { + _reverbOptions = &_scriptReverbOptions; + reverbChanged = true; + } + + if (reverbChanged) { + configureReverb(); + } +} + +template +void AudioPacketHandler::setReverb(bool reverb) { + _reverb = reverb; + + if (!_reverb) { + _sourceReverb.reset(); + _listenerReverb.reset(); + _localReverb.reset(); + } +} + +template +void AudioPacketHandler::setReverbOptions(const AudioEffectOptions* options) { + // Save the new options + _scriptReverbOptions.setBandwidth(options->getBandwidth()); + _scriptReverbOptions.setPreDelay(options->getPreDelay()); + _scriptReverbOptions.setLateDelay(options->getLateDelay()); + _scriptReverbOptions.setReverbTime(options->getReverbTime()); + _scriptReverbOptions.setEarlyDiffusion(options->getEarlyDiffusion()); + _scriptReverbOptions.setLateDiffusion(options->getLateDiffusion()); + _scriptReverbOptions.setRoomSize(options->getRoomSize()); + _scriptReverbOptions.setDensity(options->getDensity()); + _scriptReverbOptions.setBassMult(options->getBassMult()); + _scriptReverbOptions.setBassFreq(options->getBassFreq()); + _scriptReverbOptions.setHighGain(options->getHighGain()); + _scriptReverbOptions.setHighFreq(options->getHighFreq()); + _scriptReverbOptions.setModRate(options->getModRate()); + _scriptReverbOptions.setModDepth(options->getModDepth()); + _scriptReverbOptions.setEarlyGain(options->getEarlyGain()); + _scriptReverbOptions.setLateGain(options->getLateGain()); + _scriptReverbOptions.setEarlyMixLeft(options->getEarlyMixLeft()); + _scriptReverbOptions.setEarlyMixRight(options->getEarlyMixRight()); + _scriptReverbOptions.setLateMixLeft(options->getLateMixLeft()); + _scriptReverbOptions.setLateMixRight(options->getLateMixRight()); + _scriptReverbOptions.setWetDryMix(options->getWetDryMix()); + + if (_reverbOptions == &_scriptReverbOptions) { + // Apply them to the reverb instances + configureReverb(); + } +} + +#if defined(WEBRTC_AUDIO) + +static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = *src++; + f *= (1/32768.0f); // scale + dst[ch][i] = f; // deinterleave + } + } +} + +static void interleaveToInt16(const float* const* src, int16_t* dst, int numFrames, int numChannels) { + for (int i = 0; i < numFrames; i++) { + for (int ch = 0; ch < numChannels; ch++) { + float f = src[ch][i]; + f *= 32768.0f; // scale + f += (f < 0.0f) ? -0.5f : 0.5f; // round + f = std::max(std::min(f, 32767.0f), -32768.0f); // saturate + *dst++ = (int16_t)f; // interleave + } + } +} + +template +void AudioPacketHandler::configureWebrtc() { + _apm = webrtc::AudioProcessingBuilder().Create(); + + webrtc::AudioProcessing::Config config; + + config.pre_amplifier.enabled = false; + config.high_pass_filter.enabled = false; + config.echo_canceller.enabled = true; + config.echo_canceller.mobile_mode = false; +#if defined(WEBRTC_LEGACY) + config.echo_canceller.use_legacy_aec = false; +#endif + config.noise_suppression.enabled = false; + config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; + config.voice_detection.enabled = false; + config.gain_controller1.enabled = false; + config.gain_controller2.enabled = false; + config.gain_controller2.fixed_digital.gain_db = 0.0f; + config.gain_controller2.adaptive_digital.enabled = false; + config.residual_echo_detector.enabled = true; + config.level_estimation.enabled = false; + + _apm->ApplyConfig(config); +} + +// rebuffer into 10ms chunks +template +void AudioPacketHandler::processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + const int numChunk = (int)streamConfig.num_frames(); + + static int32_t lastWarningHash = 0; + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX || numChannels > WEBRTC_CHANNELS_MAX) { + if (lastWarningHash != ((sampleRate << 8) | numChannels)) { + lastWarningHash = ((sampleRate << 8) | numChannels); + qCWarning(audioclient) << "AEC not unsupported for output format: sampleRate =" << sampleRate << "numChannels =" << numChannels; + } + return; + } + + while (numFrames > 0) { + + // number of frames to fill + int numFill = std::min(numFrames, numChunk - _numFifoFarEnd); + + // refill fifo + memcpy(&_fifoFarEnd[_numFifoFarEnd], samples, numFill * numChannels * sizeof(int16_t)); + samples += numFill * numChannels; + numFrames -= numFill; + _numFifoFarEnd += numFill; + + if (_numFifoFarEnd == numChunk) { + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(_fifoFarEnd, buffers, numChunk, numChannels); + + // process one chunk + int error = _apm->ProcessReverseStream(buffers, streamConfig, streamConfig, buffers); + if (error != _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessReverseStream() returned ERROR:" << error; + } + _numFifoFarEnd = 0; + } + } +} + +template +void AudioPacketHandler::processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate) { + + const webrtc::StreamConfig streamConfig = webrtc::StreamConfig(sampleRate, numChannels); + assert(numFrames == (int)streamConfig.num_frames()); // WebRTC requires exactly 10ms of input + + static int32_t lastWarningHash = 0; + if (sampleRate > WEBRTC_SAMPLE_RATE_MAX || numChannels > WEBRTC_CHANNELS_MAX) { + if (lastWarningHash != ((sampleRate << 8) | numChannels)) { + lastWarningHash = ((sampleRate << 8) | numChannels); + qCWarning(audioclient) << "AEC not unsupported for input format: sampleRate =" << sampleRate << "numChannels =" << numChannels; + } + return; + } + + // convert audio format + float buffer[WEBRTC_CHANNELS_MAX][WEBRTC_FRAMES_MAX]; + float* const buffers[WEBRTC_CHANNELS_MAX] = { buffer[0], buffer[1] }; + deinterleaveToFloat(samples, buffers, numFrames, numChannels); + + // process one chunk + int error = _apm->ProcessStream(buffers, streamConfig, streamConfig, buffers); + if (error != _apm->kNoError) { + qCWarning(audioclient) << "WebRTC ProcessStream() returned ERROR:" << error; + } else { + // modify samples in-place + interleaveToInt16(buffers, samples, numFrames, numChannels); + } +} + +#endif // WEBRTC_AUDIO + +template +float AudioPacketHandler::loudnessToLevel(float loudness) { + float level = loudness * (1 / 32768.0f); // level in [0, 1] + level = 6.02059991f * fastLog2f(level); // convert to dBFS + level = (level + 48.0f) * (1 / 42.0f); // map [-48, -6] dBFS to [0, 1] + return glm::clamp(level, 0.0f, 1.0f); +} + +template +void AudioPacketHandler::handleAudioInput(QByteArray& audioBuffer) { + if (!_audioPaused) { + + bool audioGateOpen = false; + + if (!_isMuted) { + int16_t* samples = reinterpret_cast(audioBuffer.data()); + int numSamples = audioBuffer.size() / AudioConstants::SAMPLE_SIZE; + int numFrames = numSamples / _desiredInputFormat.channelCount; + + if (_isNoiseGateEnabled && _isNoiseReductionAutomatic) { + // The audio gate includes DC removal + audioGateOpen = _audioGate->render(samples, samples, numFrames); + } else if (_isNoiseGateEnabled && !_isNoiseReductionAutomatic && + loudnessToLevel(_lastSmoothedRawInputLoudness) >= _noiseReductionThreshold) { + audioGateOpen = _audioGate->removeDC(samples, samples, numFrames); + } else if (_isNoiseGateEnabled && !_isNoiseReductionAutomatic) { + audioGateOpen = false; + } else { + audioGateOpen = _audioGate->removeDC(samples, samples, numFrames); + } + + // FIXME: CRTP + // emit inputReceived(audioBuffer); + } + + // loudness after mute/gate + _lastInputLoudness = (_isMuted || !audioGateOpen) ? 0.0f : _lastRawInputLoudness; + + // detect gate opening and closing + bool openedInLastBlock = !_audioGateOpen && audioGateOpen; // the gate just opened + bool closedInLastBlock = _audioGateOpen && !audioGateOpen; // the gate just closed + _audioGateOpen = audioGateOpen; + + if (openedInLastBlock) { + //FIXME: CRTP + // emit noiseGateOpened(); + } else if (closedInLastBlock) { + //FIXME: CRTP + // emit noiseGateClosed(); + } + + // the codec must be flushed to silence before sending silent packets, + // so delay the transition to silent packets by one packet after becoming silent. + auto packetType = _shouldEchoToServer ? PacketType::MicrophoneAudioWithEcho : PacketType::MicrophoneAudioNoEcho; + if (!audioGateOpen && !closedInLastBlock) { + packetType = PacketType::SilentAudioFrame; + _silentOutbound.increment(); + } else { + _audioOutbound.increment(); + } + + QByteArray encodedBuffer; + if (_encoder) { + _encoder->encode(audioBuffer, encodedBuffer); + } else { + encodedBuffer = audioBuffer; + } + + AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, _desiredInputFormat.channelCount == AudioConstants::STEREO, + {_positionGetter(), _orientationGetter()}, avatarBoundingBoxCorner, avatarBoundingBoxScale, + packetType, _selectedCodecName); + _stats.sentPacket(); + } +} + +// CRTP: replaced libraries/audio-client/AudioClient::handleMicAudioInput +// void AudioClient::handleMicAudioInput() { +// if (!_inputDevice || _isPlayingBackRecording) { +// return; +// } +// QByteArray inputByteArray = _inputDevice->readAll(); +// +// #if defined(Q_OS_ANDROID) +// _inputReadsSinceLastCheck++; +// #endif +// +// handleLocalEchoAndReverb(inputByteArray); +// +// handleMicAudioInput(inputByteArray.data(), inputByteArray.size()); +// sendInput(); +// } + +template +void AudioPacketHandler::handleMicAudioInput(const char* data, int size) { + + const auto sampleBytes = _inputFormat.getSampleSize(); + const auto trailingBytes = size % sampleBytes; + if (trailingBytes != 0) { + qCWarning(audioclient) << "Input buffer trailing bytes: " << trailingBytes; + size -= trailingBytes; // we ignore trailing bytes + } + + if (_inputFormat.sampleType == AudioFormat::Float) { + _inputTypeConversionBuffer.resize(size / sizeof(float)); + const auto floats = reinterpret_cast(data); + for (int i = 0; i < (int)_inputTypeConversionBuffer.size(); i+= _inputFormat.channelCount) { + for (int j = 0; j < _inputFormat.channelCount; ++j) { + _inputTypeConversionBuffer[i+j] = convertToSignedInt(floats[i+j]); + } + } + + data = reinterpret_cast(_inputTypeConversionBuffer.data()); + size = _inputTypeConversionBuffer.size() * sizeof(int16_t); + } + + _inputRingBuffer.writeData(data, size); + + float audioInputMsecsRead = size / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); + _stats.updateInputMsRead(audioInputMsecsRead); +} + +template +void AudioPacketHandler::sendInput() { + const int numNetworkBytes = _desiredInputFormat.channelCount == AudioConstants::STEREO + ? AudioConstants::NETWORK_FRAME_BYTES_STEREO + : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + const int numNetworkSamples = _desiredInputFormat.channelCount == AudioConstants::STEREO + ? AudioConstants::NETWORK_FRAME_SAMPLES_STEREO + : AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL; + + // input samples required to produce exactly NETWORK_FRAME_SAMPLES of output + const int inputSamplesRequired = (_inputToNetworkResampler ? + _inputToNetworkResampler->getMinInput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) : + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * _inputFormat.channelCount; + + static int16_t networkAudioSamples[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO]; + while (_inputRingBuffer.samplesAvailable() >= inputSamplesRequired) { + + // TODO: this should at least be an std::vector to avoid + // unnecessary re-allocations, and better resized in + // setupInput() not here. + const auto inputAudioSamples = std::unique_ptr(new int16_t[inputSamplesRequired]); + + _inputRingBuffer.readSamples(inputAudioSamples.get(), inputSamplesRequired); + + // detect clipping on the raw input + bool isClipping = detectClipping(inputAudioSamples.get(), inputSamplesRequired, _inputFormat.channelCount); + if (isClipping) { + _timeSinceLastClip = 0.0f; + } else if (_timeSinceLastClip >= 0.0f) { + _timeSinceLastClip += AudioConstants::NETWORK_FRAME_SECS; + } + isClipping = (_timeSinceLastClip >= 0.0f) && (_timeSinceLastClip < 2.0f); // 2 second hold time + +#if defined(WEBRTC_AUDIO) + if (_isAECEnabled) { + processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount, + _inputFormat.channelCount, _inputFormat.sampleRate); + } +#endif + + float loudness = computeLoudness(inputAudioSamples.get(), inputSamplesRequired); + _lastRawInputLoudness = loudness; + + // envelope detection + float tc = (loudness > _lastSmoothedRawInputLoudness) ? 0.378f : 0.967f; // 10ms attack, 300ms release @ 100Hz + loudness += tc * (_lastSmoothedRawInputLoudness - loudness); + _lastSmoothedRawInputLoudness = loudness; + + // FIXME: CRTP + // emit inputLoudnessChanged(_lastSmoothedRawInputLoudness, isClipping); + + if (!_isMuted) { + possibleResampling(_inputToNetworkResampler, + inputAudioSamples.get(), networkAudioSamples, + inputSamplesRequired, numNetworkSamples, + _inputFormat.channelCount, _desiredInputFormat.channelCount); + } + int bytesInInputRingBuffer = _inputRingBuffer.samplesAvailable() * AudioConstants::SAMPLE_SIZE; + float msecsInInputRingBuffer = bytesInInputRingBuffer / (float)(_inputFormat.bytesForDuration(USECS_PER_MSEC)); + _stats.updateInputMsUnplayed(msecsInInputRingBuffer); + + QByteArray audioBuffer(reinterpret_cast(networkAudioSamples), numNetworkBytes); + handleAudioInput(audioBuffer); + } +} + +template +void AudioPacketHandler::handleRecordedAudioInput(const QByteArray& audio) { + QByteArray audioBuffer(audio); + handleAudioInput(audioBuffer); +} + +template +void AudioPacketHandler::prepareLocalAudioInjectors(std::unique_ptr localAudioLock) { + bool doSynchronously = localAudioLock.operator bool(); + if (!localAudioLock) { + localAudioLock.reset(new Lock(_localAudioMutex)); + } + + int samplesNeeded = std::numeric_limits::max(); + while (samplesNeeded > 0) { + if (!doSynchronously) { + // unlock between every write to allow device switching + localAudioLock->unlock(); + localAudioLock->lock(); + } + + // in case of a device switch, consider bufferCapacity volatile across iterations + if (_outputPeriod == 0) { + return; + } + + int bufferCapacity = _localInjectorsStream.getSampleCapacity(); + int maxOutputSamples = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * AudioConstants::STEREO; + if (_localToOutputResampler) { + maxOutputSamples = + _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL) * + AudioConstants::STEREO; + } + + samplesNeeded = bufferCapacity - _localSamplesAvailable.load(std::memory_order_relaxed); + if (samplesNeeded < maxOutputSamples) { + // avoid overwriting the buffer to prevent losing frames + break; + } + + // get a network frame of local injectors' audio + if (!mixLocalAudioInjectors(_localMixBuffer)) { + break; + } + + // reverb + if (_reverb) { + _localReverb.render(_localMixBuffer, _localMixBuffer, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } + + int samples; + if (_localToOutputResampler) { + // resample to output sample rate + int frames = _localToOutputResampler->render(_localMixBuffer, _localOutputMixBuffer, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + + // write to local injectors' ring buffer + samples = frames * AudioConstants::STEREO; + _localInjectorsStream.writeSamples(_localOutputMixBuffer, samples); + + } else { + // write to local injectors' ring buffer + samples = AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + _localInjectorsStream.writeSamples(_localMixBuffer, + AudioConstants::NETWORK_FRAME_SAMPLES_STEREO); + } + + _localSamplesAvailable.fetch_add(samples, std::memory_order_release); + samplesNeeded -= samples; + } +} + +template +bool AudioPacketHandler::mixLocalAudioInjectors(float* mixBuffer) { + // check the flag for injectors before attempting to lock + if (!_localInjectorsAvailable.load(std::memory_order_acquire)) { + return false; + } + + // lock the injectors + Lock lock(_injectorsMutex); + + QVector injectorsToRemove; + + memset(mixBuffer, 0, AudioConstants::NETWORK_FRAME_SAMPLES_STEREO * sizeof(float)); + + for (const AudioInjectorPointer& injector : _activeLocalAudioInjectors) { + // the lock guarantees that injectorBuffer, if found, is invariant + auto injectorBuffer = injector->getLocalBuffer(); + if (injectorBuffer) { + + auto options = injector->getOptions(); + + static const int HRTF_DATASET_INDEX = 1; + + int numChannels = options.ambisonic ? AudioConstants::AMBISONIC : (options.stereo ? AudioConstants::STEREO : AudioConstants::MONO); + size_t bytesToRead = numChannels * AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + + // get one frame from the injector + memset(_localScratchBuffer, 0, bytesToRead); + if (0 < injectorBuffer->readData((char*)_localScratchBuffer, bytesToRead)) { + + bool isSystemSound = !options.positionSet && !options.ambisonic; + + float gain = options.volume * (isSystemSound ? _systemInjectorGain : _localInjectorGain); + + if (options.ambisonic) { + + if (options.positionSet) { + + // distance attenuation + glm::vec3 relativePosition = options.position - _positionGetter(); + float distance = glm::max(glm::length(relativePosition), EPSILON); + gain = gainForSource(distance, gain); + } + + // + // Calculate the soundfield orientation relative to the listener. + // Injector orientation can be used to align a recording to our world coordinates. + // + glm::quat relativeOrientation = options.orientation * glm::inverse(_orientationGetter()); + + // convert from Y-up (OpenGL) to Z-up (Ambisonic) coordinate system + float qw = relativeOrientation.w; + float qx = -relativeOrientation.z; + float qy = -relativeOrientation.x; + float qz = relativeOrientation.y; + + // spatialize into mixBuffer + injector->getLocalFOA().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, + qw, qx, qy, qz, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } else if (options.stereo) { + + if (options.positionSet) { + + // distance attenuation + glm::vec3 relativePosition = options.position - _positionGetter(); + float distance = glm::max(glm::length(relativePosition), EPSILON); + gain = gainForSource(distance, gain); + } + + // direct mix into mixBuffer + injector->getLocalHRTF().mixStereo(_localScratchBuffer, mixBuffer, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } else { // injector is mono + + if (options.positionSet) { + + // distance attenuation + glm::vec3 relativePosition = options.position - _positionGetter(); + float distance = glm::max(glm::length(relativePosition), EPSILON); + gain = gainForSource(distance, gain); + + float azimuth = azimuthForSource(relativePosition); + + // spatialize into mixBuffer + injector->getLocalHRTF().render(_localScratchBuffer, mixBuffer, HRTF_DATASET_INDEX, + azimuth, distance, gain, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } else { + + // direct mix into mixBuffer + injector->getLocalHRTF().mixMono(_localScratchBuffer, mixBuffer, gain, + AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } + } + + } else { + + //qCDebug(audioclient) << "injector has no more data, marking finished for removal"; + injector->finishLocalInjection(); + injectorsToRemove.append(injector); + } + + } else { + + //qCDebug(audioclient) << "injector has no local buffer, marking as finished for removal"; + injector->finishLocalInjection(); + injectorsToRemove.append(injector); + } + } + + for (const AudioInjectorPointer& injector : injectorsToRemove) { + //qCDebug(audioclient) << "removing injector"; + _activeLocalAudioInjectors.removeOne(injector); + } + + // update the flag + _localInjectorsAvailable.exchange(!_activeLocalAudioInjectors.empty(), std::memory_order_release); + + return true; +} + +template +void AudioPacketHandler::processReceivedSamples(const QByteArray& decodedBuffer, QByteArray& outputBuffer) { + + const int16_t* decodedSamples = reinterpret_cast(decodedBuffer.data()); + assert(decodedBuffer.size() == AudioConstants::NETWORK_FRAME_BYTES_STEREO); + + outputBuffer.resize(_outputFrameSize * AudioConstants::SAMPLE_SIZE); + int16_t* outputSamples = reinterpret_cast(outputBuffer.data()); + + bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); + + // apply stereo reverb + if (hasReverb) { + updateReverbOptions(); + int16_t* reverbSamples = _networkToOutputResampler ? _networkScratchBuffer : outputSamples; + _listenerReverb.render(decodedSamples, reverbSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } + + // resample to output sample rate + if (_networkToOutputResampler) { + const int16_t* inputSamples = hasReverb ? _networkScratchBuffer : decodedSamples; + _networkToOutputResampler->render(inputSamples, outputSamples, AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL); + } + + // if no transformations were applied, we still need to copy the buffer + if (!hasReverb && !_networkToOutputResampler) { + memcpy(outputSamples, decodedSamples, decodedBuffer.size()); + } +} + +template +void AudioPacketHandler::sendMuteEnvironmentPacket() { + auto nodeList = DependencyManager::get(); + + int dataSize = sizeof(glm::vec3) + sizeof(float); + + auto mutePacket = NLPacket::create(PacketType::MuteEnvironment, dataSize); + + const float MUTE_RADIUS = 50; + + glm::vec3 currentSourcePosition = _positionGetter(); + + mutePacket->writePrimitive(currentSourcePosition); + mutePacket->writePrimitive(MUTE_RADIUS); + + // grab our audio mixer from the NodeList, if it exists + SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); + + if (audioMixer) { + // send off this mute packet + nodeList->sendPacket(std::move(mutePacket), *audioMixer); + } +} + +template +void AudioPacketHandler::setMuted(bool muted, bool emitSignal) { + if (_isMuted != muted) { + _isMuted = muted; + if (emitSignal) { + // FIXME: CRTP + // emit muteToggled(_isMuted); + } + } +} + +template +void AudioPacketHandler::setNoiseReduction(bool enable, bool emitSignal) { + if (_isNoiseGateEnabled != enable) { + _isNoiseGateEnabled = enable; + if (emitSignal) { + // FIXME: CRTP + // emit noiseReductionChanged(_isNoiseGateEnabled); + } + } +} + +template +void AudioPacketHandler::setNoiseReductionAutomatic(bool enable, bool emitSignal) { + if (_isNoiseReductionAutomatic != enable) { + _isNoiseReductionAutomatic = enable; + if (emitSignal) { + // FIXME: CRTP + // emit noiseReductionAutomaticChanged(_isNoiseReductionAutomatic); + } + } +} + +template +void AudioPacketHandler::setNoiseReductionThreshold(float threshold, bool emitSignal) { + if (_noiseReductionThreshold != threshold) { + _noiseReductionThreshold = threshold; + if (emitSignal) { + // FIXME: CRTP + // emit noiseReductionThresholdChanged(_noiseReductionThreshold); + } + } +} + +template +void AudioPacketHandler::setAcousticEchoCancellation(bool enable, bool emitSignal) { + if (_isAECEnabled != enable) { + _isAECEnabled = enable; + if (emitSignal) { + // FIXME: CRTP + // emit acousticEchoCancellationChanged(_isAECEnabled); + } + } +} + +template +bool AudioPacketHandler::outputLocalInjector(const AudioInjectorPointer& injector) { + auto injectorBuffer = injector->getLocalBuffer(); + if (injectorBuffer) { + // local injectors are on the AudioInjectorsThread, so we must guard access + Lock lock(_injectorsMutex); + if (!_activeLocalAudioInjectors.contains(injector)) { + //qCDebug(audioclient) << "adding new injector"; + _activeLocalAudioInjectors.append(injector); + + // update the flag + _localInjectorsAvailable.exchange(true, std::memory_order_release); + } else { + qCDebug(audioclient) << "injector exists in active list already"; + } + + return true; + + } else { + // no local buffer + return false; + } +} + +template +int AudioPacketHandler::getNumLocalInjectors() { + Lock lock(_injectorsMutex); + return _activeLocalAudioInjectors.size(); +} + +template +void AudioPacketHandler::outputFormatChanged() { + _outputFrameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * OUTPUT_CHANNEL_COUNT * _outputFormat.sampleRate) / + _desiredOutputFormat.sampleRate; + _receivedAudioStream.outputFormatChanged(_outputFormat.sampleRate, OUTPUT_CHANNEL_COUNT); + + _audioOutputIODevice.open(); + + // FIXEM: CRTP + // // setup our general output device for audio-mixer audio + // _audioOutput = new QAudioOutput(_outputDeviceInfo.getDevice(), _outputFormat, this); + // + // _audioOutput->setBufferSize(_outputPeriod * 16); // magic number introduced with commit 47af406440daaccdc48bf01cb3669233a940f811 + // + // connect(_audioOutput, &QAudioOutput::notify, this, &AudioClient::outputNotify); + // + // // start the output device + // _audioOutput->start(&_audioOutputIODevice); + // + // // initialize mix buffers + // + // // restrict device callback to _outputPeriod samples + // _outputPeriod = _audioOutput->periodSize() / AudioConstants::SAMPLE_SIZE; + // // device callback may exceed reported period, so double it to avoid stutter + // _outputPeriod *= 2; + +} + +template +void AudioPacketHandler::cleanupInput() { + +// FIXME: CRTP +// if (_loopbackResampler) { +// delete _loopbackResampler; +// _loopbackResampler = NULL; +// } +// +// // cleanup any previously initialized device +// if (_audioInput) { +// // The call to stop() causes _inputDevice to be destructed. +// // That in turn causes it to be disconnected (see for example +// // http://stackoverflow.com/questions/9264750/qt-signals-and-slots-object-disconnect). +// _audioInput->stop(); +// _inputDevice = NULL; +// +// _audioInput->deleteLater(); +// _audioInput = NULL; +// +// _inputDeviceInfo.setDevice(QAudioDeviceInfo()); +// } +// +// +// #if defined(Q_OS_ANDROID) +// _shouldRestartInputSetup = false; // avoid a double call to _audioInput->start() from audioInputStateChanged +// #endif + + + if (_dummyAudioInput) { + _dummyAudioInput->stop(); + _dummyAudioInput->deleteLater(); + _dummyAudioInput = nullptr; + } + + // cleanup any resamplers + if (_inputToNetworkResampler) { + delete _inputToNetworkResampler; + _inputToNetworkResampler = NULL; + } + + if (_audioGate) { + delete _audioGate; + _audioGate = nullptr; + } + +} + +template +bool AudioPacketHandler::setupInput(AudioFormat inputFormat) { + bool supportedFormat = false; + + if (inputFormat.channelCount > 2) { + qCDebug(audioclient) << "Audio input has too many channels: " << inputFormat.channelCount; + return false; + } + + _inputFormat = inputFormat; + + // if necessary reinitialize the codec + if (_inputFormat.channelCount != _desiredInputFormat.channelCount) { + if (_codec) { + if (_encoder) { + _codec->releaseEncoder(_encoder); + } + _encoder = _codec->createEncoder(_desiredInputFormat.sampleRate, _inputFormat.channelCount); + qCDebug(audioclient) << "Reset Codec:" << _selectedCodecName << "isStereoInput:" << (_inputFormat.channelCount == 2); + } + _desiredInputFormat.channelCount = inputFormat.channelCount; + } + + qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat; + + // we've got the best we can get for input + // if required, setup a resampler for this input to our desired network format + if (_inputFormat != _desiredInputFormat + && _inputFormat.sampleRate != _desiredInputFormat.sampleRate) { + qCDebug(audioclient) << "Attemping to create a resampler for input format to network format."; + + assert(_desiredInputFormat.getSampleBits() == 16); + int channelCount = (_inputFormat.channelCount == 2 && _desiredInputFormat.channelCount == 2) ? 2 : 1; + + _inputToNetworkResampler = new AudioSRC(_inputFormat.sampleRate, _desiredInputFormat.sampleRate, channelCount); + + } else { + qCDebug(audioclient) << "No resampling required for audio input to match desired network format."; + } + + // the audio gate runs after the resampler + _audioGate = new AudioGate(_desiredInputFormat.sampleRate, _desiredInputFormat.channelCount); + qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount << "channels."; + auto numInputCallbackBytes = calculateNumberOfInputCallbackBytes(_inputFormat); + // how do we want to handle input working, but output not working? + int numFrameSamples = calculateNumberOfFrameSamples(numInputCallbackBytes); + _inputRingBuffer.resizeForFrameSize(numFrameSamples); + + return supportedFormat; +} + +template +void AudioPacketHandler::setupDummyInput() { + // Generates audio callbacks on a timer to simulate a mic stream of silent packets. + // This enables clients without a mic to still receive an audio stream from the mixer. + + // FIXME: CRTP + // qCDebug(audioclient) << "Audio input device is not available, using dummy input."; + // _inputDeviceInfo.setDevice(QAudioDeviceInfo()); + // emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo); + + _inputFormat = _desiredInputFormat; + qCDebug(audioclient) << "The format to be used for audio input is" << _inputFormat; + qCDebug(audioclient) << "No re-sampling required for audio input to match desired network format."; + + _audioGate = new AudioGate(_desiredInputFormat.sampleRate, _desiredInputFormat.channelCount); + qCDebug(audioclient) << "Noise gate created with" << _desiredInputFormat.channelCount << "channels."; + + // generate audio callbacks at the network sample rate + _dummyAudioInput = new QTimer(); + _dummyAudioInput->connect(_dummyAudioInput, &QTimer::timeout, [this](){ + const int numNetworkBytes = _desiredInputFormat.channelCount == AudioConstants::STEREO + ? AudioConstants::NETWORK_FRAME_BYTES_STEREO + : AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL; + + QByteArray audioBuffer(numNetworkBytes, 0); // silent + handleAudioInput(audioBuffer); + }); + _dummyAudioInput->start((int)(AudioConstants::NETWORK_FRAME_MSECS + 0.5f)); +} + +template +bool AudioPacketHandler::isDummyInput() { + return _dummyAudioInput != nullptr; +} + +// FIXME: CRTP move to derived, different implementation +// bool AudioClient::switchInputToAudioDevice(const HifiAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) { +// Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread"); +// +// qCDebug(audioclient) << __FUNCTION__ << "_inputDeviceInfo: [" << _inputDeviceInfo.deviceName() << ":" << _inputDeviceInfo.getDevice().deviceName() +// << "-- inputDeviceInfo:" << inputDeviceInfo.deviceName() << ":" << inputDeviceInfo.getDevice().deviceName() << "]"; +// // NOTE: device start() uses the Qt internal device list +// Lock lock(_deviceMutex); +// +// bool supportedFormat = false; +// +// cleanupInput(); +// +// if (!inputDeviceInfo.getDevice().isNull()) { +// qCDebug(audioclient) << "The audio input device" << inputDeviceInfo.deviceName() << ":" << inputDeviceInfo.getDevice().deviceName() << "is available."; +// +// //do not update UI that we're changing devices if default or same device +// _inputDeviceInfo = inputDeviceInfo; +// emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo); +// +// QAudioFormat format; +// if (adjustedFormatForAudioDevice(_inputDeviceInfo.getDevice(), _desiredInputFormat, format)) { +// setupInput(format); +// +// auto numInputCallbackBytes = calculateNumberOfInputCallbackBytes(_inputFormat); +// // if the user wants stereo but this device can't provide then bail +// if (!_isStereoInput || _inputFormat.channelCount() == 2) { +// _audioInput = new QAudioInput(_inputDeviceInfo.getDevice(), _inputFormat, this); +// _audioInput->setBufferSize(numInputCallbackBytes * CALLBACK_ACCELERATOR_RATIO); +// // different audio input devices may have different volumes +// emit inputVolumeChanged(_audioInput->volume()); +// +// #if defined(Q_OS_ANDROID) +// if (_audioInput) { +// _shouldRestartInputSetup = true; +// connect(_audioInput, &QAudioInput::stateChanged, this, &AudioClient::audioInputStateChanged); +// } +// #endif +// _inputDevice = _audioInput->start(); +// +// if (_inputDevice) { +// connect(_inputDevice, SIGNAL(readyRead()), this, SLOT(handleMicAudioInput())); +// supportedFormat = true; +// } else { +// qCDebug(audioclient) << "Error starting audio input -" << _audioInput->error(); +// _audioInput->deleteLater(); +// _audioInput = NULL; +// } +// } +// } +// } +// +// // If there is no working input device, use the dummy input device. +// if (!_audioInput) { +// setupDummyInput(); +// } +// +// return supportedFormat; +// } + +template +void AudioPacketHandler::cleanupOutput() { + // FIXME: CRTP + // cleanup any previously initialized device + // if (_audioOutput) { + // _audioOutput->stop(); + // + // //must be deleted in next eventloop cycle when its called from notify() + // _audioOutput->deleteLater(); + // _audioOutput = NULL; + // + // _loopbackOutputDevice = NULL; + // //must be deleted in next eventloop cycle when its called from notify() + // _loopbackAudioOutput->deleteLater(); + // _loopbackAudioOutput = NULL; + // + // _outputDeviceInfo.setDevice(QAudioDeviceInfo()); + // } + // + // if (_loopbackResampler) { + // delete _loopbackResampler; + // _loopbackResampler = NULL; + // } + + _audioOutputInitialized = false; + + if (_audioOutputIODevice.isOpen()) { + _audioOutputIODevice.close(); + } + + if (_outputMixBuffer) { + delete[] _outputMixBuffer; + _outputMixBuffer = NULL; + + delete[] _outputScratchBuffer; + _outputScratchBuffer = NULL; + + delete[] _localOutputMixBuffer; + _localOutputMixBuffer = NULL; + } + + // cleanup any resamplers + if (_networkToOutputResampler) { + delete _networkToOutputResampler; + _networkToOutputResampler = NULL; + } + + if (_localToOutputResampler) { + delete _localToOutputResampler; + _localToOutputResampler = NULL; + } +} + +template +bool AudioPacketHandler::setupOutput(AudioFormat outputFormat) { + _outputFormat = outputFormat; + + qCDebug(audioclient) << "The format to be used for audio output is" << _outputFormat; + + // we've got the best we can get for input + // if required, setup a resampler for this input to our desired network format + if (_desiredOutputFormat != _outputFormat + && _desiredOutputFormat.sampleRate != _outputFormat.sampleRate) { + qCDebug(audioclient) << "Attemping to create a resampler for network format to output format."; + + assert(_desiredOutputFormat.getSampleBits() == 16); + + _networkToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate, _outputFormat.sampleRate, OUTPUT_CHANNEL_COUNT); + _localToOutputResampler = new AudioSRC(_desiredOutputFormat.sampleRate, _outputFormat.sampleRate, OUTPUT_CHANNEL_COUNT); + + } else { + qCDebug(audioclient) << "No resampling required for network output to match actual output format."; + } + + + int frameSize = (AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * _outputFormat.channelCount * _outputFormat.sampleRate) / _desiredOutputFormat.sampleRate; + int requestedSize = _sessionOutputBufferSizeFrames * frameSize * AudioConstants::SAMPLE_SIZE; + + _outputPeriod = requestedSize; + + outputFormatChanged(); + + _outputMixBuffer = new float[_outputPeriod]; + _outputScratchBuffer = new int16_t[_outputPeriod]; + + // size local output mix buffer based on resampled network frame size + int networkPeriod = _localToOutputResampler ? _localToOutputResampler->getMaxOutput(AudioConstants::NETWORK_FRAME_SAMPLES_STEREO) : AudioConstants::NETWORK_FRAME_SAMPLES_STEREO; + _localOutputMixBuffer = new float[networkPeriod]; + + // local period should be at least twice the output period, + // in case two device reads happen before more data can be read (worst case) + int localPeriod = _outputPeriod * 2; + // round up to an exact multiple of networkPeriod + localPeriod = ((localPeriod + networkPeriod - 1) / networkPeriod) * networkPeriod; + // this ensures lowest latency without stutter from underrun + _localInjectorsStream.resizeForFrameSize(localPeriod); + + _audioOutputInitialized = true; + + return true; + +} + +// FIXME: CRTP +// bool AudioClient::switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) { +// Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread"); +// +// qCDebug(audioclient) << __FUNCTION__ << "_outputdeviceInfo: [" << _outputDeviceInfo.deviceName() << ":" << _outputDeviceInfo.getDevice().deviceName() +// << "-- outputDeviceInfo:" << outputDeviceInfo.deviceName() << ":" << outputDeviceInfo.getDevice().deviceName() << "]"; +// bool supportedFormat = false; +// +// // NOTE: device start() uses the Qt internal device list +// Lock lock(_deviceMutex); +// +// _localSamplesAvailable.exchange(0, std::memory_order_release); +// +// //wait on local injectors prep to finish running +// if ( !_localPrepInjectorFuture.isFinished()) { +// _localPrepInjectorFuture.waitForFinished(); +// } +// +// Lock localAudioLock(_localAudioMutex); +// +// cleanupOutput(); +// +// if (!outputDeviceInfo.getDevice().isNull()) { +// qCDebug(audioclient) << "The audio output device" << outputDeviceInfo.deviceName() << ":" << outputDeviceInfo.getDevice().deviceName() << "is available."; +// +// //do not update UI that we're changing devices if default or same device +// _outputDeviceInfo = outputDeviceInfo; +// emit deviceChanged(QAudio::AudioOutput, _outputDeviceInfo); +// +// AudioFormat outputFormat; +// if (adjustedFormatForAudioDevice(_outputDeviceInfo.getDevice(), _desiredOutputFormat, outputFormat)) { +// setupOutput(outputFormat); +// +// int bufferSize = _audioOutput->bufferSize(); +// int bufferSamples = bufferSize / AudioConstants::SAMPLE_SIZE; +// int bufferFrames = bufferSamples / (float)frameSize; +// qCDebug(audioclient) << "frame (samples):" << frameSize; +// qCDebug(audioclient) << "buffer (frames):" << bufferFrames; +// qCDebug(audioclient) << "buffer (samples):" << bufferSamples; +// qCDebug(audioclient) << "buffer (bytes):" << bufferSize; +// qCDebug(audioclient) << "requested (bytes):" << requestedSize; +// qCDebug(audioclient) << "period (samples):" << _outputPeriod; +// qCDebug(audioclient) << "local buffer (samples):" << localPeriod; +// +// // unlock to avoid a deadlock with the device callback (which always succeeds this initialization) +// localAudioLock.unlock(); +// +// // setup a loopback audio output device +// _loopbackAudioOutput = new QAudioOutput(outputDeviceInfo.getDevice(), _outputFormat, this); +// +// supportedFormat = true; +// } +// } +// +// return supportedFormat; +// } + +template +int AudioPacketHandler::setOutputBufferSize(int numFrames, bool persist) { + qCDebug(audioclient) << __FUNCTION__ << "numFrames:" << numFrames << "persist:" << persist; + + numFrames = std::min(std::max(numFrames, MIN_BUFFER_FRAMES), MAX_BUFFER_FRAMES); + qCDebug(audioclient) << __FUNCTION__ << "clamped numFrames:" << numFrames << "_sessionOutputBufferSizeFrames:" << _sessionOutputBufferSizeFrames; + + if (numFrames != _sessionOutputBufferSizeFrames) { + qCInfo(audioclient, "Audio output buffer set to %d frames", numFrames); + _sessionOutputBufferSizeFrames = numFrames; + if (persist) { + // FIXME: CRTP + // _outputBufferSizeFrames.set(numFrames); + } + } + return numFrames; +} + +template +int AudioPacketHandler::calculateNumberOfInputCallbackBytes(const AudioFormat& format) const { + int numInputCallbackBytes = (int)(((AudioConstants::NETWORK_FRAME_BYTES_PER_CHANNEL + * format.channelCount + * ((float) format.sampleRate / AudioConstants::SAMPLE_RATE))) + 0.5f); + + return numInputCallbackBytes; +} + +template +int AudioPacketHandler::calculateNumberOfFrameSamples(int numBytes) const { + int frameSamples = numBytes / AudioConstants::SAMPLE_SIZE; + return frameSamples; +} + +template +float AudioPacketHandler::azimuthForSource(const glm::vec3& relativePosition) { + glm::quat inverseOrientation = glm::inverse(_orientationGetter()); + + glm::vec3 rotatedSourcePosition = inverseOrientation * relativePosition; + + // project the rotated source position vector onto the XZ plane + rotatedSourcePosition.y = 0.0f; + + static const float SOURCE_DISTANCE_THRESHOLD = 1e-30f; + + float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition); + if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) { + + // produce an oriented angle about the y-axis + glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2)); + float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward" + return (direction.x < 0.0f) ? -angle : angle; + + } else { + // no azimuth if they are in same spot + return 0.0f; + } +} + +template +float AudioPacketHandler::gainForSource(float distance, float volume) { + + // attenuation = -6dB * log2(distance) + // reference attenuation of 0dB at distance = ATTN_DISTANCE_REF + float d = (1.0f / ATTN_DISTANCE_REF) * std::max(distance, HRTF_NEARFIELD_MIN); + float gain = volume / d; + gain = std::min(gain, ATTN_GAIN_MAX); + + return gain; +} + +template +qint64 AudioPacketHandler::AudioOutputIODevice::readData(char * data, qint64 maxSize) { + + // lock-free wait for initialization to avoid races + if (!_audio->_audioOutputInitialized.load(std::memory_order_acquire)) { + memset(data, 0, maxSize); + return maxSize; + } + + // max samples requested from OUTPUT_CHANNEL_COUNT + int deviceChannelCount = _audio->_outputFormat.channelCount; + int maxSamplesRequested = (int)(maxSize / _audio->_outputFormat.getSampleSize()) * OUTPUT_CHANNEL_COUNT / deviceChannelCount; + // restrict samplesRequested to the size of our mix/scratch buffers + maxSamplesRequested = std::min(maxSamplesRequested, _audio->_outputPeriod); + + int16_t* scratchBuffer = _audio->_outputScratchBuffer; + float* mixBuffer = _audio->_outputMixBuffer; + + int samplesRequested = maxSamplesRequested; + int networkSamplesPopped; + if ((networkSamplesPopped = _receivedAudioStream.popSamples(samplesRequested, false)) > 0) { + qCDebug(audiostream, "Read %d samples from buffer (%d available, %d requested)", networkSamplesPopped, _receivedAudioStream.getSamplesAvailable(), samplesRequested); + AudioRingBuffer::ConstIterator lastPopOutput = _receivedAudioStream.getLastPopOutput(); + lastPopOutput.readSamples(scratchBuffer, networkSamplesPopped); + for (int i = 0; i < networkSamplesPopped; i++) { + mixBuffer[i] = convertToFloat(scratchBuffer[i]); + } + samplesRequested = networkSamplesPopped; + } + + int injectorSamplesPopped = 0; + { + bool append = networkSamplesPopped > 0; + // check the samples we have available locklessly; this is possible because only two functions add to the count: + // - prepareLocalAudioInjectors will only increase samples count + // - switchOutputToAudioDevice will zero samples count, + // stop the device - so that readData will exhaust the existing buffer or see a zeroed samples count, + // and start the device - which can then only see a zeroed samples count + int samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire); + + // if we do not have enough samples buffered despite having injectors, buffer them synchronously + if (samplesAvailable < samplesRequested && _audio->_localInjectorsAvailable.load(std::memory_order_acquire)) { + // try_to_lock, in case the device is being shut down already + std::unique_ptr localAudioLock(new Lock(_audio->_localAudioMutex, std::try_to_lock)); + if (localAudioLock->owns_lock()) { + _audio->prepareLocalAudioInjectors(std::move(localAudioLock)); + samplesAvailable = _audio->_localSamplesAvailable.load(std::memory_order_acquire); + } + } + + samplesRequested = std::min(samplesRequested, samplesAvailable); + if ((injectorSamplesPopped = _localInjectorsStream.appendSamples(mixBuffer, samplesRequested, append)) > 0) { + _audio->_localSamplesAvailable.fetch_sub(injectorSamplesPopped, std::memory_order_release); + qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); + } + } + + // prepare injectors for the next callback + _audio->_localPrepInjectorFuture = QtConcurrent::run(QThreadPool::globalInstance(), [this] { + _audio->prepareLocalAudioInjectors(); + }); + + int samplesPopped = std::max(networkSamplesPopped, injectorSamplesPopped); + if (samplesPopped == 0) { + // nothing on network, don't grab anything from injectors, and fill with silence + samplesPopped = maxSamplesRequested; + memset(mixBuffer, 0, samplesPopped * sizeof(float)); + } + int framesPopped = samplesPopped / OUTPUT_CHANNEL_COUNT; + + // apply output gain + float newGain = _audio->_outputGain.load(std::memory_order_acquire); + float oldGain = _audio->_lastOutputGain; + _audio->_lastOutputGain = newGain; + + applyGainSmoothing(mixBuffer, framesPopped, oldGain, newGain); + + // limit the audio + _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); + +#if defined(WEBRTC_AUDIO) + if (_audio->_isAECEnabled) { + _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate); + } +#endif + + int bytesWritten = 0; + + if (_audio->_outputFormat.sampleType == AudioFormat::Float) { + // Convert samples to normalized float. This is used in the client + // library, since a lot of audio APIs prefer to restrict themselves to + // float samples. + for (int i = 0; i < samplesPopped; i++) { + mixBuffer[i] = convertToFloat(scratchBuffer[i]); + } + + // if required, upmix or downmix to deviceChannelCount + bytesWritten = copyWithChannelConversion(mixBuffer, OUTPUT_CHANNEL_COUNT, data, deviceChannelCount, samplesPopped); + } + else + { + // if required, upmix or downmix to deviceChannelCount + bytesWritten = copyWithChannelConversion(scratchBuffer, OUTPUT_CHANNEL_COUNT, data, deviceChannelCount, samplesPopped); + } + + assert(bytesWritten <= maxSize); + + // FIXME: CRTP + // send output buffer for recording + // if (_audio->_isRecording) { + // Lock lock(_recordMutex); + // _audio->_audioFileWav.addRawAudioChunk(data, bytesWritten); + // } + + // FIXEM: CRTP + // int bytesAudioOutputUnplayed = _audio->_audioOutput->bufferSize() - _audio->_audioOutput->bytesFree(); + // float msecsAudioOutputUnplayed = bytesAudioOutputUnplayed / (float)_audio->_outputFormat.bytesForDuration(USECS_PER_MSEC); + // _audio->_stats.updateOutputMsUnplayed(msecsAudioOutputUnplayed); + // + // if (bytesAudioOutputUnplayed == 0) { + // _unfulfilledReads++; + // } + + return bytesWritten; +} + +template +void AudioPacketHandler::setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale) { + avatarBoundingBoxCorner = corner; + avatarBoundingBoxScale = scale; +} + + +template +void AudioPacketHandler::startThread() { + moveToNewNamedThread(&derived(), "Audio Thread", [this] { start(); }, QThread::TimeCriticalPriority); +} + +#endif /* end of include guard */ diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt index d82e94bd695..46a6f976423 100644 --- a/libraries/audio-client/CMakeLists.txt +++ b/libraries/audio-client/CMakeLists.txt @@ -8,8 +8,7 @@ include_hifi_library_headers(shared) include_hifi_library_headers(shared-gui) include_hifi_library_headers(networking) -if (ANDROID) -else () +if (ENABLE_WEBRTC_AUDIO) target_webrtc() endif () diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index eb6820421de..52c77c036ab 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -95,7 +95,7 @@ void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { // thread-safe QList getAvailableDevices(QAudio::Mode mode, const QString& hmdName) { //get hmd device name prior to locking device mutex. in case of shutdown, this thread will be locked and audio client - //cannot properly shut down. + //cannot properly shut down. QString defDeviceName = defaultAudioDeviceName(mode); // NOTE: availableDevices() clobbers the Qt internal device list @@ -132,7 +132,7 @@ QList getAvailableDevices(QAudio::Mode mode, const QString& break; } } - + if (!hmdDevice.getDevice().isNull()) { newDevices.push_front(hmdDevice); } @@ -159,7 +159,7 @@ void AudioClient::checkDevices() { auto inputDevices = getAvailableDevices(QAudio::AudioInput, hmdInputName); auto outputDevices = getAvailableDevices(QAudio::AudioOutput, hmdOutputName); - + static const QMetaMethod devicesChangedSig= QMetaMethod::fromSignal(&AudioClient::devicesChanged); //only emit once the scripting interface has connected to the signal if (isSignalConnected(devicesChangedSig)) { @@ -173,7 +173,7 @@ void AudioClient::checkDevices() { _outputDevices.swap(outputDevices); emit devicesChanged(QAudio::AudioOutput, _outputDevices); } - } + } } HifiAudioDeviceInfo AudioClient::getActiveAudioDevice(QAudio::Mode mode) const { @@ -280,7 +280,7 @@ static float computeLoudness(int16_t* samples, int numSamples) { template static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float gain1) { - + // fast path for unity gain if (gain0 == 1.0f && gain1 == 1.0f) { return; @@ -295,7 +295,7 @@ static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float float tStep = 1.0f / numFrames; for (int i = 0; i < numFrames; i++) { - + // evaluate poly over t=[0,1) float gain = (c3 * t + c2) * t * t + c0; t += tStep; @@ -386,7 +386,7 @@ AudioClient::AudioClient() { PacketReceiver::makeUnsourcedListenerReference(this, &AudioClient::handleSelectedAudioFormat)); auto& domainHandler = nodeList->getDomainHandler(); - connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { + connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { _solo.reset(); }); connect(nodeList.data(), &NodeList::nodeActivated, this, [this](SharedNodePointer node) { @@ -578,7 +578,7 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { waveInGetDevCaps(WAVE_MAPPER, &wic, sizeof(wic)); //Use the received manufacturer id to get the device's real name waveInGetDevCaps(wic.wMid, &wic, sizeof(wic)); -#if !defined(NDEBUG) +#if !defined(NDEBUG) qCDebug(audioclient) << "input device:" << wic.szPname; #endif deviceName = wic.szPname; @@ -588,7 +588,7 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { waveOutGetDevCaps(WAVE_MAPPER, &woc, sizeof(woc)); //Use the received manufacturer id to get the device's real name waveOutGetDevCaps(woc.wMid, &woc, sizeof(woc)); -#if !defined(NDEBUG) +#if !defined(NDEBUG) qCDebug(audioclient) << "output device:" << woc.szPname; #endif deviceName = woc.szPname; @@ -613,8 +613,8 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { CoUninitialize(); } -#if !defined(NDEBUG) - qCDebug(audioclient) << "defaultAudioDeviceForMode mode: " << (mode == QAudio::AudioOutput ? "Output" : "Input") +#if !defined(NDEBUG) + qCDebug(audioclient) << "defaultAudioDeviceForMode mode: " << (mode == QAudio::AudioOutput ? "Output" : "Input") << " [" << deviceName << "] [" << "]"; #endif @@ -795,7 +795,7 @@ void AudioClient::start() { _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); - + QString inputName; QString outputName; { @@ -803,10 +803,10 @@ void AudioClient::start() { inputName = _hmdInputName; outputName = _hmdOutputName; } - + //initialize input to the dummy device to prevent starves switchInputToAudioDevice(HifiAudioDeviceInfo()); - switchOutputToAudioDevice(defaultAudioDeviceForMode(QAudio::AudioOutput, QString())); + switchOutputToAudioDevice(defaultAudioDeviceForMode(QAudio::AudioOutput, QString())); #if defined(Q_OS_ANDROID) connect(&_checkInputTimer, &QTimer::timeout, this, &AudioClient::checkInputTimeout); @@ -1015,7 +1015,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { bool AudioClient::switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo) { auto device = deviceInfo; if (deviceInfo.getDevice().isNull()) { - qCDebug(audioclient) << __FUNCTION__ << " switching to null device :" + qCDebug(audioclient) << __FUNCTION__ << " switching to null device :" << deviceInfo.deviceName() << " : " << deviceInfo.getDevice().deviceName(); } @@ -1401,10 +1401,6 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { _audioOutbound.increment(); } - Transform audioTransform; - audioTransform.setTranslation(_positionGetter()); - audioTransform.setRotation(_orientationGetter()); - QByteArray encodedBuffer; if (_encoder) { _encoder->encode(audioBuffer, encodedBuffer); @@ -1413,7 +1409,7 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { } emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), _outgoingAvatarAudioSequenceNumber, _isStereoInput, - audioTransform, avatarBoundingBoxCorner, avatarBoundingBoxScale, + {_positionGetter(), _orientationGetter()}, avatarBoundingBoxCorner, avatarBoundingBoxScale, packetType, _selectedCodecName); _stats.sentPacket(); } @@ -1868,7 +1864,7 @@ void AudioClient::outputFormatChanged() { bool AudioClient::switchInputToAudioDevice(const HifiAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) { Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread"); - qCDebug(audioclient) << __FUNCTION__ << "_inputDeviceInfo: [" << _inputDeviceInfo.deviceName() << ":" << _inputDeviceInfo.getDevice().deviceName() + qCDebug(audioclient) << __FUNCTION__ << "_inputDeviceInfo: [" << _inputDeviceInfo.deviceName() << ":" << _inputDeviceInfo.getDevice().deviceName() << "-- inputDeviceInfo:" << inputDeviceInfo.deviceName() << ":" << inputDeviceInfo.getDevice().deviceName() << "]"; bool supportedFormat = false; @@ -1923,7 +1919,7 @@ bool AudioClient::switchInputToAudioDevice(const HifiAudioDeviceInfo inputDevice if (!inputDeviceInfo.getDevice().isNull()) { qCDebug(audioclient) << "The audio input device" << inputDeviceInfo.deviceName() << ":" << inputDeviceInfo.getDevice().deviceName() << "is available."; - + //do not update UI that we're changing devices if default or same device _inputDeviceInfo = inputDeviceInfo; emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo); @@ -2097,13 +2093,13 @@ void AudioClient::outputNotify() { void AudioClient::noteAwakening() { qCDebug(audioclient) << "Restarting the audio devices."; - switchInputToAudioDevice(_inputDeviceInfo); + switchInputToAudioDevice(_inputDeviceInfo); switchOutputToAudioDevice(_outputDeviceInfo); } bool AudioClient::switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) { Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread"); - + qCDebug(audioclient) << __FUNCTION__ << "_outputdeviceInfo: [" << _outputDeviceInfo.deviceName() << ":" << _outputDeviceInfo.getDevice().deviceName() << "-- outputDeviceInfo:" << outputDeviceInfo.deviceName() << ":" << outputDeviceInfo.getDevice().deviceName() << "]"; bool supportedFormat = false; @@ -2143,7 +2139,7 @@ bool AudioClient::switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDevi delete[] _localOutputMixBuffer; _localOutputMixBuffer = NULL; - + _outputDeviceInfo.setDevice(QAudioDeviceInfo()); } @@ -2170,7 +2166,7 @@ bool AudioClient::switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDevi if (!outputDeviceInfo.getDevice().isNull()) { qCDebug(audioclient) << "The audio output device" << outputDeviceInfo.deviceName() << ":" << outputDeviceInfo.getDevice().deviceName() << "is available."; - + //do not update UI that we're changing devices if default or same device _outputDeviceInfo = outputDeviceInfo; emit deviceChanged(QAudio::AudioOutput, _outputDeviceInfo); @@ -2398,7 +2394,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); } } - + // prepare injectors for the next callback _audio->_localPrepInjectorFuture = QtConcurrent::run(QThreadPool::globalInstance(), [this] { _audio->prepareLocalAudioInjectors(); diff --git a/libraries/audio/CMakeLists.txt b/libraries/audio/CMakeLists.txt index ae9035e2267..d1b177fd854 100644 --- a/libraries/audio/CMakeLists.txt +++ b/libraries/audio/CMakeLists.txt @@ -5,4 +5,4 @@ if (ANDROID) add_definitions("-D__STDC_CONSTANT_MACROS") endif () -link_hifi_libraries(networking shared shared-gui plugins) +link_hifi_libraries(networking shared) diff --git a/libraries/audio/src/AbstractAudioInterface.cpp b/libraries/audio/src/AbstractAudioInterface.cpp index 99e6ef38704..33bc2dc74ef 100644 --- a/libraries/audio/src/AbstractAudioInterface.cpp +++ b/libraries/audio/src/AbstractAudioInterface.cpp @@ -13,12 +13,13 @@ #include #include #include -#include + +#include #include "AudioConstants.h" void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, bool isStereo, - const Transform& transform, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale, + const Vantage& vantage, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale, PacketType packetType, QString codecName) { static std::mutex _mutex; using Locker = std::unique_lock; @@ -48,12 +49,12 @@ void AbstractAudioInterface::emitAudioPacket(const void* audioData, size_t bytes } // at this point we'd better be sending the mixer a valid position, or it won't consider us for mixing - assert(!isNaN(transform.getTranslation())); + assert(!isNaN(vantage.position)); // pack the three float positions - audioPacket->writePrimitive(transform.getTranslation()); + audioPacket->writePrimitive(vantage.position); // pack the orientation - audioPacket->writePrimitive(transform.getRotation()); + audioPacket->writePrimitive(vantage.rotation); audioPacket->writePrimitive(avatarBoundingBoxCorner); audioPacket->writePrimitive(avatarBoundingBoxScale); diff --git a/libraries/audio/src/AbstractAudioInterface.h b/libraries/audio/src/AbstractAudioInterface.h index e9e40e95f94..5d293ee37f0 100644 --- a/libraries/audio/src/AbstractAudioInterface.h +++ b/libraries/audio/src/AbstractAudioInterface.h @@ -23,7 +23,11 @@ class AudioInjector; class AudioInjectorLocalBuffer; -class Transform; + +struct Vantage { + glm::vec3 position; + glm::quat rotation; +}; class AbstractAudioInterface : public QObject { Q_OBJECT @@ -31,7 +35,7 @@ class AbstractAudioInterface : public QObject { AbstractAudioInterface(QObject* parent = 0) : QObject(parent) {}; static void emitAudioPacket(const void* audioData, size_t bytes, quint16& sequenceNumber, bool isStereo, - const Transform& transform, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale, + const Vantage& vantage, glm::vec3 avatarBoundingBoxCorner, glm::vec3 avatarBoundingBoxScale, PacketType packetType, QString codecName = QString("")); // threadsafe diff --git a/libraries/audio/src/AudioInjectorOptions.cpp b/libraries/audio/src/AudioInjectorOptions.cpp index 39b807fd769..59bf4ade690 100644 --- a/libraries/audio/src/AudioInjectorOptions.cpp +++ b/libraries/audio/src/AudioInjectorOptions.cpp @@ -11,10 +11,6 @@ #include "AudioInjectorOptions.h" -#include - -#include - #include "AudioLogging.h" AudioInjectorOptions::AudioInjectorOptions() : @@ -32,98 +28,3 @@ AudioInjectorOptions::AudioInjectorOptions() : { } -QScriptValue injectorOptionsToScriptValue(QScriptEngine* engine, const AudioInjectorOptions& injectorOptions) { - QScriptValue obj = engine->newObject(); - if (injectorOptions.positionSet) { - obj.setProperty("position", vec3ToScriptValue(engine, injectorOptions.position)); - } - obj.setProperty("volume", injectorOptions.volume); - obj.setProperty("loop", injectorOptions.loop); - obj.setProperty("orientation", quatToScriptValue(engine, injectorOptions.orientation)); - obj.setProperty("ignorePenumbra", injectorOptions.ignorePenumbra); - obj.setProperty("localOnly", injectorOptions.localOnly); - obj.setProperty("secondOffset", injectorOptions.secondOffset); - obj.setProperty("pitch", injectorOptions.pitch); - return obj; -} - -/*@jsdoc - * Configures where and how an audio injector plays its audio. - * @typedef {object} AudioInjector.AudioInjectorOptions - * @property {Vec3} position=Vec3.ZERO - The position in the domain to play the sound. - * @property {Quat} orientation=Quat.IDENTITY - The orientation in the domain to play the sound in. - * @property {number} volume=1.0 - Playback volume, between 0.0 and 1.0. - * @property {number} pitch=1.0 - Alter the pitch of the sound, within +/- 2 octaves. The value is the relative sample rate to - * resample the sound at, range 0.0625 – 16.0.
- * A value of 0.0625 lowers the pitch by 2 octaves.
- * A value of 1.0 means there is no change in pitch.
- * A value of 16.0 raises the pitch by 2 octaves. - * @property {boolean} loop=false - If true, the sound is played repeatedly until playback is stopped. - * @property {number} secondOffset=0 - Starts playback from a specified time (seconds) within the sound file, ≥ - * 0. - * @property {boolean} localOnly=false - If true, the sound is played back locally on the client rather than to - * others via the audio mixer. - * @property {boolean} ignorePenumbra=false -

Deprecated: This property is deprecated and will be - * removed.

- */ -void injectorOptionsFromScriptValue(const QScriptValue& object, AudioInjectorOptions& injectorOptions) { - if (!object.isObject()) { - qWarning() << "Audio injector options is not an object."; - return; - } - - if (injectorOptions.positionSet == false) { - qWarning() << "Audio injector options: injectorOptionsFromScriptValue() called more than once?"; - } - injectorOptions.positionSet = false; - - QScriptValueIterator it(object); - while (it.hasNext()) { - it.next(); - - if (it.name() == "position") { - vec3FromScriptValue(object.property("position"), injectorOptions.position); - injectorOptions.positionSet = true; - } else if (it.name() == "orientation") { - quatFromScriptValue(object.property("orientation"), injectorOptions.orientation); - } else if (it.name() == "volume") { - if (it.value().isNumber()) { - injectorOptions.volume = it.value().toNumber(); - } else { - qCWarning(audio) << "Audio injector options: volume is not a number"; - } - } else if (it.name() == "loop") { - if (it.value().isBool()) { - injectorOptions.loop = it.value().toBool(); - } else { - qCWarning(audio) << "Audio injector options: loop is not a boolean"; - } - } else if (it.name() == "ignorePenumbra") { - if (it.value().isBool()) { - injectorOptions.ignorePenumbra = it.value().toBool(); - } else { - qCWarning(audio) << "Audio injector options: ignorePenumbra is not a boolean"; - } - } else if (it.name() == "localOnly") { - if (it.value().isBool()) { - injectorOptions.localOnly = it.value().toBool(); - } else { - qCWarning(audio) << "Audio injector options: localOnly is not a boolean"; - } - } else if (it.name() == "secondOffset") { - if (it.value().isNumber()) { - injectorOptions.secondOffset = it.value().toNumber(); - } else { - qCWarning(audio) << "Audio injector options: secondOffset is not a number"; - } - } else if (it.name() == "pitch") { - if (it.value().isNumber()) { - injectorOptions.pitch = it.value().toNumber(); - } else { - qCWarning(audio) << "Audio injector options: pitch is not a number"; - } - } else { - qCWarning(audio) << "Unknown audio injector option:" << it.name(); - } - } -} diff --git a/libraries/audio/src/AudioInjectorOptions.h b/libraries/audio/src/AudioInjectorOptions.h index 5dec8a02403..92ed9711b2d 100644 --- a/libraries/audio/src/AudioInjectorOptions.h +++ b/libraries/audio/src/AudioInjectorOptions.h @@ -12,8 +12,6 @@ #ifndef hifi_AudioInjectorOptions_h #define hifi_AudioInjectorOptions_h -#include - #include #include @@ -33,9 +31,4 @@ class AudioInjectorOptions { float pitch; // multiplier, where 2.0f shifts up one octave }; -Q_DECLARE_METATYPE(AudioInjectorOptions); - -QScriptValue injectorOptionsToScriptValue(QScriptEngine* engine, const AudioInjectorOptions& injectorOptions); -void injectorOptionsFromScriptValue(const QScriptValue& object, AudioInjectorOptions& injectorOptions); - #endif // hifi_AudioInjectorOptions_h diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index e946c6afb7e..005d554039c 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -144,7 +144,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { } case SequenceNumberStats::Early: { // Packet is early. Treat the packets as if all the packets between the last - // OnTime packet and this packet were lost. If we're using a codec this will + // OnTime packet and this packet were lost. If we're using a codec this will // also result in allowing the codec to interpolate lost data. Then // fall through to the "on time" logic to actually handle this packet int packetsDropped = arrivalInfo._seqDiffFromExpected; @@ -270,7 +270,7 @@ int InboundAudioStream::lostAudioData(int numPackets) { int InboundAudioStream::parseAudioData(const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; - // may block on the real-time thread, which is acceptible as + // may block on the real-time thread, which is acceptible as // parseAudioData is only called by the packet processing // thread which, while high performance, is not as sensitive to // delays as the real-time thread. @@ -287,22 +287,22 @@ int InboundAudioStream::parseAudioData(const QByteArray& packetAfterStreamProper int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { // We can't guarentee that all clients have faded the stream down - // to silence and encoded that silence before sending us a + // to silence and encoded that silence before sending us a // SilentAudioFrame. If the encoder has truncated the stream it will - // leave the decoder holding some unknown loud state. To handle this + // leave the decoder holding some unknown loud state. To handle this // case we will call the decoder's lostFrame() method, which indicates - // that it should interpolate from its last known state down toward + // that it should interpolate from its last known state down toward // silence. { - // may block on the real-time thread, which is acceptible as + // may block on the real-time thread, which is acceptible as // writeDroppableSilentFrames is only called by the packet processing // thread which, while high performance, is not as sensitive to // delays as the real-time thread. QMutexLocker lock(&_decoderMutex); if (_decoder) { - // FIXME - We could potentially use the output from the codec, in which - // case we might get a cleaner fade toward silence. NOTE: The below logic - // attempts to catch up in the event that the jitter buffers have grown. + // FIXME - We could potentially use the output from the codec, in which + // case we might get a cleaner fade toward silence. NOTE: The below logic + // attempts to catch up in the event that the jitter buffers have grown. // The better long term fix is to use the output from the decode, detect // when it actually reaches silence, and then delete the silent portions // of the jitter buffers. Or petentially do a cross fade from the decode @@ -338,7 +338,7 @@ int InboundAudioStream::writeDroppableSilentFrames(int silentFrames) { } int ret = _ringBuffer.addSilentSamples(silentSamples - numSilentFramesToDrop * samplesPerFrame); - + return ret; } @@ -360,7 +360,7 @@ int InboundAudioStream::popSamples(int maxSamples, bool allOrNothing) { popSamplesNoCheck(samplesAvailable); samplesPopped = samplesAvailable; } else { - // we can't pop any samples, set this stream to starved for jitter + // we can't pop any samples, set this stream to starved for jitter // buffer calculations. setToStarved(); _consecutiveNotMixedCount++; @@ -485,7 +485,7 @@ void InboundAudioStream::setStaticJitterBufferFrames(int staticJitterBufferFrame } void InboundAudioStream::packetReceivedUpdateTimingStats() { - + // update our timegap stats and desired jitter buffer frames if necessary // discard the first few packets we receive since they usually have gaps that aren't represensative of normal jitter const quint32 NUM_INITIAL_PACKETS_DISCARD = 1000; // 10s @@ -567,7 +567,7 @@ float calculateRepeatedFrameFadeFactor(int indexOfRepeat) { return 0.0f; } -void InboundAudioStream::setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels) { +void InboundAudioStream::setupCodec(std::shared_ptr codec, const QString& codecName, int numChannels) { cleanupCodec(); // cleanup any previously allocated coders first _codec = codec; _selectedCodecName = codecName; diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index b42609d5769..9e604e47c95 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -18,8 +18,7 @@ #include #include #include - -#include +#include #include "AudioRingBuffer.h" #include "MovingMinMaxAvg.h" @@ -75,7 +74,7 @@ class InboundAudioStream : public NodeData { /// returns the desired number of jitter buffer frames under the dyanmic jitter buffers scheme int getCalculatedJitterBufferFrames() const { return _calculatedJitterBufferFrames; } - + bool dynamicJitterBufferEnabled() const { return _dynamicJitterBufferEnabled; } int getStaticJitterBufferFrames() { return _staticJitterBufferFrames; } int getDesiredJitterBufferFrames() { return _desiredJitterBufferFrames; } @@ -95,14 +94,14 @@ class InboundAudioStream : public NodeData { int getOverflowCount() const { return _ringBuffer.getOverflowCount(); } int getPacketsReceived() const { return _incomingSequenceNumberStats.getReceived(); } - + bool hasReverb() const { return _hasReverb; } float getRevebTime() const { return _reverbTime; } float getWetLevel() const { return _wetLevel; } void setReverb(float reverbTime, float wetLevel); void clearReverb() { _hasReverb = false; } - void setupCodec(CodecPluginPointer codec, const QString& codecName, int numChannels); + void setupCodec(std::shared_ptr codec, const QString& codecName, int numChannels); void cleanupCodec(); signals: @@ -139,7 +138,7 @@ public slots: /// writes silent frames to the buffer that may be dropped to reduce latency caused by the buffer virtual int writeDroppableSilentFrames(int silentFrames); - + protected: AudioRingBuffer _ringBuffer; @@ -147,7 +146,7 @@ public slots: bool _lastPopSucceeded { false }; AudioRingBuffer::ConstIterator _lastPopOutput; - + bool _dynamicJitterBufferEnabled { DEFAULT_DYNAMIC_JITTER_BUFFER_ENABLED }; int _staticJitterBufferFrames { DEFAULT_STATIC_JITTER_FRAMES }; int _desiredJitterBufferFrames; @@ -185,7 +184,7 @@ public slots: float _reverbTime { 0.0f }; float _wetLevel { 0.0f }; - CodecPluginPointer _codec; + std::shared_ptr _codec; QString _selectedCodecName; QMutex _decoderMutex; Decoder* _decoder { nullptr }; diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 5732f32e902..c237c4e8e18 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -14,8 +14,6 @@ #include "InboundAudioStream.h" -class AudioClient; - class MixedProcessedAudioStream : public InboundAudioStream { Q_OBJECT public: diff --git a/libraries/audio/src/PositionalAudioStream.cpp b/libraries/audio/src/PositionalAudioStream.cpp index 4161b660608..4c370571661 100644 --- a/libraries/audio/src/PositionalAudioStream.cpp +++ b/libraries/audio/src/PositionalAudioStream.cpp @@ -81,7 +81,7 @@ int PositionalAudioStream::parsePositionalData(const QByteArray& positionalByteA packetStream.readRawData(reinterpret_cast(&_avatarBoundingBoxCorner), sizeof(_avatarBoundingBoxCorner)); packetStream.readRawData(reinterpret_cast(&_avatarBoundingBoxScale), sizeof(_avatarBoundingBoxScale)); - if (_avatarBoundingBoxCorner != _ignoreBox.getCorner()) { + if (_avatarBoundingBoxCorner != _ignoreBox.corner) { // if the ignore box corner changes, we need to re-calculate the ignore box calculateIgnoreBox(); } @@ -117,7 +117,7 @@ void PositionalAudioStream::calculateIgnoreBox() { scale *= IGNORE_BOX_SCALE_FACTOR; // create the box (we use a box for the zone for convenience) - _ignoreBox.setBox(_avatarBoundingBoxCorner, scale); + _ignoreBox = AABoxData{_avatarBoundingBoxCorner, scale}; } } diff --git a/libraries/audio/src/PositionalAudioStream.h b/libraries/audio/src/PositionalAudioStream.h index 01a714aeb42..6bf93bfe18b 100644 --- a/libraries/audio/src/PositionalAudioStream.h +++ b/libraries/audio/src/PositionalAudioStream.h @@ -13,7 +13,7 @@ #define hifi_PositionalAudioStream_h #include -#include +#include #include "InboundAudioStream.h" @@ -69,7 +69,7 @@ class PositionalAudioStream : public InboundAudioStream { const glm::vec3& getAvatarBoundingBoxCorner() const { return _avatarBoundingBoxCorner; } const glm::vec3& getAvatarBoundingBoxScale() const { return _avatarBoundingBoxScale; } - using IgnoreBox = AABox; + using IgnoreBox = AABoxData; // called from single AudioMixerSlave while processing packets for node void enableIgnoreBox(); diff --git a/libraries/opus-codec/CMakeLists.txt b/libraries/opus-codec/CMakeLists.txt new file mode 100644 index 00000000000..ef3467bf97c --- /dev/null +++ b/libraries/opus-codec/CMakeLists.txt @@ -0,0 +1,4 @@ +set(TARGET_NAME opus-codec) +setup_hifi_library() +link_hifi_libraries(shared audio) +target_opus() diff --git a/libraries/opus-codec/src/OpusCodec.cpp b/libraries/opus-codec/src/OpusCodec.cpp new file mode 100644 index 00000000000..3aeec062a67 --- /dev/null +++ b/libraries/opus-codec/src/OpusCodec.cpp @@ -0,0 +1,34 @@ +// +// OpusCodec.cpp +// libraries/opus-codec/src +// +// Created by Nshan G. on 3 July 2022. +// Copyright 2019 Michael Bailey +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "OpusCodec.h" +#include "OpusEncoder.h" +#include "OpusDecoder.h" + +const char* OpusCodec::NAME { "opus" }; + +Encoder* OpusCodec::createEncoder(int sampleRate, int numChannels) { + return new OpusEncoder(sampleRate, numChannels); +} + +Decoder* OpusCodec::createDecoder(int sampleRate, int numChannels) { + return new OpusDecoder(sampleRate, numChannels); +} + +void OpusCodec::releaseEncoder(Encoder* encoder) { + delete encoder; +} + +void OpusCodec::releaseDecoder(Decoder* decoder) { + delete decoder; +} diff --git a/libraries/opus-codec/src/OpusCodec.h b/libraries/opus-codec/src/OpusCodec.h new file mode 100644 index 00000000000..0ae193175de --- /dev/null +++ b/libraries/opus-codec/src/OpusCodec.h @@ -0,0 +1,35 @@ +// +// OpusCodec.h +// libraries/opus-codec/src +// +// Created by Nshan G. on 3 July 2022. +// Copyright 2019 Michael Bailey +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef LIBRARIES_OPUS_CODEC_SRC_OPUSCODEC_H +#define LIBRARIES_OPUS_CODEC_SRC_OPUSCODEC_H + +#include + +class OpusCodec : public Codec { + +public: + const QString getName() const override { return NAME; } + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + + static const char* getNameCString() { return NAME; } + +private: + static const char* NAME; +}; + +#endif /* end of include guard */ diff --git a/plugins/opusCodec/src/OpusDecoder.cpp b/libraries/opus-codec/src/OpusDecoder.cpp similarity index 89% rename from plugins/opusCodec/src/OpusDecoder.cpp rename to libraries/opus-codec/src/OpusDecoder.cpp index e3e4e3645a4..3b403802eb0 100644 --- a/plugins/opusCodec/src/OpusDecoder.cpp +++ b/libraries/opus-codec/src/OpusDecoder.cpp @@ -1,6 +1,6 @@ // -// OpusCodecManager.h -// plugins/opusCodec/src +// OpusDecoder.h +// libraries/opus-codec/src // // Copyright 2020 Dale Glass // @@ -14,7 +14,7 @@ #include "OpusDecoder.h" -static QLoggingCategory decoder("AthenaOpusDecoder"); +static QLoggingCategory decoder("OpusDecoder"); static QString error_to_string(int error) { switch (error) { @@ -38,7 +38,7 @@ static QString error_to_string(int error) { } -AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) { +OpusDecoder::OpusDecoder(int sampleRate, int numChannels) { int error; _opusSampleRate = sampleRate; @@ -56,16 +56,16 @@ AthenaOpusDecoder::AthenaOpusDecoder(int sampleRate, int numChannels) { qCDebug(decoder) << "Opus decoder initialized, sampleRate = " << sampleRate << "; numChannels = " << numChannels; } -AthenaOpusDecoder::~AthenaOpusDecoder() { +OpusDecoder::~OpusDecoder() { if (_decoder) { opus_decoder_destroy(_decoder); } } -void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &decodedBuffer) { +void OpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &decodedBuffer) { assert(_decoder); - PerformanceTimer perfTimer("AthenaOpusDecoder::decode"); + PerformanceTimer perfTimer("OpusDecoder::decode"); // The audio system encodes and decodes always in fixed size chunks int bufferSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) @@ -96,10 +96,10 @@ void AthenaOpusDecoder::decode(const QByteArray &encodedBuffer, QByteArray &deco } -void AthenaOpusDecoder::lostFrame(QByteArray &decodedBuffer) { +void OpusDecoder::lostFrame(QByteArray &decodedBuffer) { assert(_decoder); - PerformanceTimer perfTimer("AthenaOpusDecoder::lostFrame"); + PerformanceTimer perfTimer("OpusDecoder::lostFrame"); int bufferSize = AudioConstants::NETWORK_FRAME_SAMPLES_PER_CHANNEL * static_cast(sizeof(int16_t)) * _opusNumChannels; diff --git a/plugins/opusCodec/src/OpusDecoder.h b/libraries/opus-codec/src/OpusDecoder.h similarity index 63% rename from plugins/opusCodec/src/OpusDecoder.h rename to libraries/opus-codec/src/OpusDecoder.h index 095893856bf..f9d4f9cc4b0 100644 --- a/plugins/opusCodec/src/OpusDecoder.h +++ b/libraries/opus-codec/src/OpusDecoder.h @@ -1,6 +1,6 @@ // -// OpusCodecManager.h -// plugins/opusCodec/src +// OpusDecoder.h +// libraries/opus-codec/src // // Copyright 2020 Dale Glass // @@ -8,18 +8,17 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef OPUSDECODER_H -#define OPUSDECODER_H +#ifndef LIBRARIES_OPUS_CODEC_SRC_OPUSDECODER_H +#define LIBRARIES_OPUS_CODEC_SRC_OPUSDECODER_H -#include +#include #include - -class AthenaOpusDecoder : public Decoder { +class OpusDecoder : public Decoder { public: - AthenaOpusDecoder(int sampleRate, int numChannels); - ~AthenaOpusDecoder() override; + OpusDecoder(int sampleRate, int numChannels); + ~OpusDecoder() override; virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override; @@ -35,5 +34,4 @@ class AthenaOpusDecoder : public Decoder { int _decodedSize = 0; }; - -#endif // OPUSDECODER_H +#endif /* end of include guard */ diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/libraries/opus-codec/src/OpusEncoder.cpp similarity index 82% rename from plugins/opusCodec/src/OpusEncoder.cpp rename to libraries/opus-codec/src/OpusEncoder.cpp index 3408701633b..8a6d098ef46 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/libraries/opus-codec/src/OpusEncoder.cpp @@ -8,13 +8,13 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include "OpusEncoder.h" + #include #include #include -#include "OpusEncoder.h" - -static QLoggingCategory encoder("AthenaOpusEncoder"); +static QLoggingCategory encoder("OpusEncoder"); static QString errorToString(int error) { switch (error) { @@ -39,7 +39,7 @@ static QString errorToString(int error) { -AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) { +OpusEncoder::OpusEncoder(int sampleRate, int numChannels) { _opusSampleRate = sampleRate; _opusChannels = numChannels; @@ -61,15 +61,15 @@ AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) { qCDebug(encoder) << "Opus encoder initialized, sampleRate = " << sampleRate << "; numChannels = " << numChannels; } -AthenaOpusEncoder::~AthenaOpusEncoder() { +OpusEncoder::~OpusEncoder() { opus_encoder_destroy(_encoder); } -void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { +void OpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) { - PerformanceTimer perfTimer("AthenaOpusEncoder::encode"); + PerformanceTimer perfTimer("OpusEncoder::encode"); assert(_encoder); encodedBuffer.resize(decodedBuffer.size()); @@ -89,14 +89,14 @@ void AthenaOpusEncoder::encode(const QByteArray& decodedBuffer, QByteArray& enco } -int AthenaOpusEncoder::getComplexity() const { +int OpusEncoder::getComplexity() const { assert(_encoder); int returnValue; opus_encoder_ctl(_encoder, OPUS_GET_COMPLEXITY(&returnValue)); return returnValue; } -void AthenaOpusEncoder::setComplexity(int complexity) { +void OpusEncoder::setComplexity(int complexity) { assert(_encoder); int returnValue = opus_encoder_ctl(_encoder, OPUS_SET_COMPLEXITY(complexity)); @@ -105,14 +105,14 @@ void AthenaOpusEncoder::setComplexity(int complexity) { } } -int AthenaOpusEncoder::getBitrate() const { +int OpusEncoder::getBitrate() const { assert(_encoder); int returnValue; opus_encoder_ctl(_encoder, OPUS_GET_BITRATE(&returnValue)); return returnValue; } -void AthenaOpusEncoder::setBitrate(int bitrate) { +void OpusEncoder::setBitrate(int bitrate) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_BITRATE(bitrate)); @@ -121,14 +121,14 @@ void AthenaOpusEncoder::setBitrate(int bitrate) { } } -int AthenaOpusEncoder::getVBR() const { +int OpusEncoder::getVBR() const { assert(_encoder); int returnValue; opus_encoder_ctl(_encoder, OPUS_GET_VBR(&returnValue)); return returnValue; } -void AthenaOpusEncoder::setVBR(int vbr) { +void OpusEncoder::setVBR(int vbr) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_VBR(vbr)); @@ -137,14 +137,14 @@ void AthenaOpusEncoder::setVBR(int vbr) { } } -int AthenaOpusEncoder::getVBRConstraint() const { +int OpusEncoder::getVBRConstraint() const { assert(_encoder); int returnValue; opus_encoder_ctl(_encoder, OPUS_GET_VBR_CONSTRAINT(&returnValue)); return returnValue; } -void AthenaOpusEncoder::setVBRConstraint(int vbr_const) { +void OpusEncoder::setVBRConstraint(int vbr_const) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_VBR_CONSTRAINT(vbr_const)); @@ -153,14 +153,14 @@ void AthenaOpusEncoder::setVBRConstraint(int vbr_const) { } } -int AthenaOpusEncoder::getMaxBandwidth() const { +int OpusEncoder::getMaxBandwidth() const { assert(_encoder); int returnValue; opus_encoder_ctl(_encoder, OPUS_GET_MAX_BANDWIDTH(&returnValue)); return returnValue; } -void AthenaOpusEncoder::setMaxBandwidth(int maxBandwidth) { +void OpusEncoder::setMaxBandwidth(int maxBandwidth) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_MAX_BANDWIDTH(maxBandwidth)); @@ -169,14 +169,14 @@ void AthenaOpusEncoder::setMaxBandwidth(int maxBandwidth) { } } -int AthenaOpusEncoder::getBandwidth() const { +int OpusEncoder::getBandwidth() const { assert(_encoder); int bandwidth; opus_encoder_ctl(_encoder, OPUS_GET_BANDWIDTH(&bandwidth)); return bandwidth; } -void AthenaOpusEncoder::setBandwidth(int bandwidth) { +void OpusEncoder::setBandwidth(int bandwidth) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_BANDWIDTH(bandwidth)); @@ -185,14 +185,14 @@ void AthenaOpusEncoder::setBandwidth(int bandwidth) { } } -int AthenaOpusEncoder::getSignal() const { +int OpusEncoder::getSignal() const { assert(_encoder); int signal; opus_encoder_ctl(_encoder, OPUS_GET_SIGNAL(&signal)); return signal; } -void AthenaOpusEncoder::setSignal(int signal) { +void OpusEncoder::setSignal(int signal) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_SIGNAL(signal)); @@ -201,14 +201,14 @@ void AthenaOpusEncoder::setSignal(int signal) { } } -int AthenaOpusEncoder::getApplication() const { +int OpusEncoder::getApplication() const { assert(_encoder); int applicationValue; opus_encoder_ctl(_encoder, OPUS_GET_APPLICATION(&applicationValue)); return applicationValue; } -void AthenaOpusEncoder::setApplication(int application) { +void OpusEncoder::setApplication(int application) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(application)); @@ -217,21 +217,21 @@ void AthenaOpusEncoder::setApplication(int application) { } } -int AthenaOpusEncoder::getLookahead() const { +int OpusEncoder::getLookahead() const { assert(_encoder); int lookAhead; opus_encoder_ctl(_encoder, OPUS_GET_LOOKAHEAD(&lookAhead)); return lookAhead; } -int AthenaOpusEncoder::getInbandFEC() const { +int OpusEncoder::getInbandFEC() const { assert(_encoder); int fec; opus_encoder_ctl(_encoder, OPUS_GET_INBAND_FEC(&fec)); return fec; } -void AthenaOpusEncoder::setInbandFEC(int inBandFEC) { +void OpusEncoder::setInbandFEC(int inBandFEC) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(inBandFEC)); @@ -240,14 +240,14 @@ void AthenaOpusEncoder::setInbandFEC(int inBandFEC) { } } -int AthenaOpusEncoder::getExpectedPacketLossPercentage() const { +int OpusEncoder::getExpectedPacketLossPercentage() const { assert(_encoder); int lossPercentage; opus_encoder_ctl(_encoder, OPUS_GET_PACKET_LOSS_PERC(&lossPercentage)); return lossPercentage; } -void AthenaOpusEncoder::setExpectedPacketLossPercentage(int percentage) { +void OpusEncoder::setExpectedPacketLossPercentage(int percentage) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(percentage)); @@ -256,14 +256,14 @@ void AthenaOpusEncoder::setExpectedPacketLossPercentage(int percentage) { } } -int AthenaOpusEncoder::getDTX() const { +int OpusEncoder::getDTX() const { assert(_encoder); int dtx; opus_encoder_ctl(_encoder, OPUS_GET_DTX(&dtx)); return dtx; } -void AthenaOpusEncoder::setDTX(int dtx) { +void OpusEncoder::setDTX(int dtx) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_DTX(dtx)); diff --git a/plugins/opusCodec/src/OpusEncoder.h b/libraries/opus-codec/src/OpusEncoder.h similarity index 82% rename from plugins/opusCodec/src/OpusEncoder.h rename to libraries/opus-codec/src/OpusEncoder.h index 10640bf409f..247a612ac0f 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/libraries/opus-codec/src/OpusEncoder.h @@ -1,6 +1,6 @@ // -// OpusCodecManager.h -// plugins/opusCodec/src +// OpusEncoder.h +// libraries/opus-codec/src // // Copyright 2020 Dale Glass // @@ -8,17 +8,17 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // -#ifndef OPUSENCODER_H -#define OPUSENCODER_H -#include -#include +#ifndef LIBRARIES_OPUS_CODEC_SRC_OPUSENCODER_H +#define LIBRARIES_OPUS_CODEC_SRC_OPUSENCODER_H +#include +#include -class AthenaOpusEncoder : public Encoder { +class OpusEncoder : public Encoder { public: - AthenaOpusEncoder(int sampleRate, int numChannels); - ~AthenaOpusEncoder() override; + OpusEncoder(int sampleRate, int numChannels); + ~OpusEncoder() override; virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; @@ -74,5 +74,4 @@ class AthenaOpusEncoder : public Encoder { OpusEncoder* _encoder = nullptr; }; - -#endif // OPUSENCODER_H +#endif /* end of include guard */ diff --git a/libraries/pcm-codec/CMakeLists.txt b/libraries/pcm-codec/CMakeLists.txt new file mode 100644 index 00000000000..93ddf68feb0 --- /dev/null +++ b/libraries/pcm-codec/CMakeLists.txt @@ -0,0 +1,3 @@ +set(TARGET_NAME pcm-codec) +setup_hifi_library() +link_hifi_libraries(shared audio) diff --git a/libraries/pcm-codec/src/PCMCodec.cpp b/libraries/pcm-codec/src/PCMCodec.cpp new file mode 100644 index 00000000000..d64ceb41c15 --- /dev/null +++ b/libraries/pcm-codec/src/PCMCodec.cpp @@ -0,0 +1,51 @@ +// +// PCMCodec.cpp +// libraries/pcm-codec/src +// +// Created by Nshan G. on 3 July 2022. +// Copyright 2016 High Fidelity, Inc. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "PCMCodec.h" + +const char* PCMCodec::NAME { "pcm" }; + +Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) { + return this; +} + +Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) { + return this; +} + +void PCMCodec::releaseEncoder(Encoder* encoder) { + // do nothing +} + +void PCMCodec::releaseDecoder(Decoder* decoder) { + // do nothing +} + +const char* zLibCodec::NAME { "zlib" }; + +Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) { + return this; +} + +Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) { + return this; +} + +void zLibCodec::releaseEncoder(Encoder* encoder) { + // do nothing... it wasn't allocated +} + +void zLibCodec::releaseDecoder(Decoder* decoder) { + // do nothing... it wasn't allocated +} + diff --git a/libraries/pcm-codec/src/PCMCodec.h b/libraries/pcm-codec/src/PCMCodec.h new file mode 100644 index 00000000000..9dff8a2a2c5 --- /dev/null +++ b/libraries/pcm-codec/src/PCMCodec.h @@ -0,0 +1,80 @@ +// +// PCMCodec.h +// libraries/pcm-codec/src +// +// Created by Nshan G. on 3 July 2022. +// Copyright 2016 High Fidelity, Inc. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef LIBRARIES_PCM_CODEC_SRC_PCMCODEC_H +#define LIBRARIES_PCM_CODEC_SRC_PCMCODEC_H + +#include +#include + +#include + +class PCMCodec : public Codec, public Encoder, public Decoder { + +public: + const QString getName() const override { return NAME; } + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = decodedBuffer; + } + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = encodedBuffer; + } + + virtual void lostFrame(QByteArray& decodedBuffer) override { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + + static const char* getNameCString() { return NAME; } + +private: + static const char* NAME; +}; + +class zLibCodec : public Codec, public Encoder, public Decoder { + +public: + const QString getName() const override { return NAME; } + + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; + virtual Decoder* createDecoder(int sampleRate, int numChannels) override; + virtual void releaseEncoder(Encoder* encoder) override; + virtual void releaseDecoder(Decoder* decoder) override; + + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { + encodedBuffer = qCompress(decodedBuffer); + } + + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { + decodedBuffer = qUncompress(encodedBuffer); + } + + virtual void lostFrame(QByteArray& decodedBuffer) override { + decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); + memset(decodedBuffer.data(), 0, decodedBuffer.size()); + } + + static const char* getNameCString() { return NAME; } + +private: + static const char* NAME; +}; + +#endif /* end of include guard */ diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index cb5b857be8e..52f20574158 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -10,26 +10,14 @@ // #pragma once +#include + #include "Plugin.h" -class Encoder { +class CodecPlugin : public Plugin, public Codec { public: - virtual ~Encoder() { } - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; + const QString getName() const override { + return static_cast(this)->getName(); + } }; -class Decoder { -public: - virtual ~Decoder() { } - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; - - virtual void lostFrame(QByteArray& decodedBuffer) = 0; -}; - -class CodecPlugin : public Plugin { -public: - virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; - virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; - virtual void releaseEncoder(Encoder* encoder) = 0; - virtual void releaseDecoder(Decoder* decoder) = 0; -}; diff --git a/libraries/plugins/src/plugins/Plugin.h b/libraries/plugins/src/plugins/Plugin.h index a0494ba6d8b..e6751706bbc 100644 --- a/libraries/plugins/src/plugins/Plugin.h +++ b/libraries/plugins/src/plugins/Plugin.h @@ -17,6 +17,7 @@ class Plugin : public QObject { Q_OBJECT public: + // NOTE: for audio plugins this is a unique identifier used in format negotiation network packets /// \return human-readable name virtual const QString getName() const = 0; diff --git a/libraries/script-engine/src/AudioScriptingInterface.cpp b/libraries/script-engine/src/AudioScriptingInterface.cpp index a55cac292f8..f8e7500e367 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.cpp +++ b/libraries/script-engine/src/AudioScriptingInterface.cpp @@ -12,12 +12,111 @@ #include "AudioScriptingInterface.h" #include +#include #include +#include +#include #include "ScriptAudioInjector.h" #include "ScriptEngineLogging.h" +QScriptValue injectorOptionsToScriptValue(QScriptEngine* engine, const AudioInjectorOptions& injectorOptions) { + QScriptValue obj = engine->newObject(); + if (injectorOptions.positionSet) { + obj.setProperty("position", vec3ToScriptValue(engine, injectorOptions.position)); + } + obj.setProperty("volume", injectorOptions.volume); + obj.setProperty("loop", injectorOptions.loop); + obj.setProperty("orientation", quatToScriptValue(engine, injectorOptions.orientation)); + obj.setProperty("ignorePenumbra", injectorOptions.ignorePenumbra); + obj.setProperty("localOnly", injectorOptions.localOnly); + obj.setProperty("secondOffset", injectorOptions.secondOffset); + obj.setProperty("pitch", injectorOptions.pitch); + return obj; +} + +/*@jsdoc + * Configures where and how an audio injector plays its audio. + * @typedef {object} AudioInjector.AudioInjectorOptions + * @property {Vec3} position=Vec3.ZERO - The position in the domain to play the sound. + * @property {Quat} orientation=Quat.IDENTITY - The orientation in the domain to play the sound in. + * @property {number} volume=1.0 - Playback volume, between 0.0 and 1.0. + * @property {number} pitch=1.0 - Alter the pitch of the sound, within +/- 2 octaves. The value is the relative sample rate to + * resample the sound at, range 0.0625 – 16.0.
+ * A value of 0.0625 lowers the pitch by 2 octaves.
+ * A value of 1.0 means there is no change in pitch.
+ * A value of 16.0 raises the pitch by 2 octaves. + * @property {boolean} loop=false - If true, the sound is played repeatedly until playback is stopped. + * @property {number} secondOffset=0 - Starts playback from a specified time (seconds) within the sound file, ≥ + * 0. + * @property {boolean} localOnly=false - If true, the sound is played back locally on the client rather than to + * others via the audio mixer. + * @property {boolean} ignorePenumbra=false -

Deprecated: This property is deprecated and will be + * removed.

+ */ +void injectorOptionsFromScriptValue(const QScriptValue& object, AudioInjectorOptions& injectorOptions) { + if (!object.isObject()) { + qWarning() << "Audio injector options is not an object."; + return; + } + + if (injectorOptions.positionSet == false) { + qWarning() << "Audio injector options: injectorOptionsFromScriptValue() called more than once?"; + } + injectorOptions.positionSet = false; + + QScriptValueIterator it(object); + while (it.hasNext()) { + it.next(); + + if (it.name() == "position") { + vec3FromScriptValue(object.property("position"), injectorOptions.position); + injectorOptions.positionSet = true; + } else if (it.name() == "orientation") { + quatFromScriptValue(object.property("orientation"), injectorOptions.orientation); + } else if (it.name() == "volume") { + if (it.value().isNumber()) { + injectorOptions.volume = it.value().toNumber(); + } else { + qCWarning(audio) << "Audio injector options: volume is not a number"; + } + } else if (it.name() == "loop") { + if (it.value().isBool()) { + injectorOptions.loop = it.value().toBool(); + } else { + qCWarning(audio) << "Audio injector options: loop is not a boolean"; + } + } else if (it.name() == "ignorePenumbra") { + if (it.value().isBool()) { + injectorOptions.ignorePenumbra = it.value().toBool(); + } else { + qCWarning(audio) << "Audio injector options: ignorePenumbra is not a boolean"; + } + } else if (it.name() == "localOnly") { + if (it.value().isBool()) { + injectorOptions.localOnly = it.value().toBool(); + } else { + qCWarning(audio) << "Audio injector options: localOnly is not a boolean"; + } + } else if (it.name() == "secondOffset") { + if (it.value().isNumber()) { + injectorOptions.secondOffset = it.value().toNumber(); + } else { + qCWarning(audio) << "Audio injector options: secondOffset is not a number"; + } + } else if (it.name() == "pitch") { + if (it.value().isNumber()) { + injectorOptions.pitch = it.value().toNumber(); + } else { + qCWarning(audio) << "Audio injector options: pitch is not a number"; + } + } else { + qCWarning(audio) << "Unknown audio injector option:" << it.name(); + } + } +} + void registerAudioMetaTypes(QScriptEngine* engine) { qScriptRegisterMetaType(engine, injectorOptionsToScriptValue, injectorOptionsFromScriptValue); qScriptRegisterMetaType(engine, soundSharedPointerToScriptValue, soundSharedPointerFromScriptValue); @@ -29,7 +128,7 @@ void AudioScriptingInterface::setLocalAudioInterface(AbstractAudioInterface* aud disconnect(_localAudioInterface, &AbstractAudioInterface::isStereoInputChanged, this, &AudioScriptingInterface::isStereoInputChanged); } - + _localAudioInterface = audioInterface; if (_localAudioInterface) { diff --git a/libraries/script-engine/src/AudioScriptingInterface.h b/libraries/script-engine/src/AudioScriptingInterface.h index 6bfb7352ee8..b48f07f3529 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.h +++ b/libraries/script-engine/src/AudioScriptingInterface.h @@ -15,6 +15,8 @@ #ifndef hifi_AudioScriptingInterface_h #define hifi_AudioScriptingInterface_h +#include + #include #include #include @@ -22,6 +24,12 @@ class ScriptAudioInjector; +Q_DECLARE_METATYPE(AudioInjectorOptions); + +QScriptValue injectorOptionsToScriptValue(QScriptEngine* engine, const AudioInjectorOptions& injectorOptions); +void injectorOptionsFromScriptValue(const QScriptValue& object, AudioInjectorOptions& injectorOptions); + + /// Provides the Audio scripting API class AudioScriptingInterface : public QObject, public Dependency { Q_OBJECT @@ -45,7 +53,7 @@ class AudioScriptingInterface : public QObject, public Dependency { } /*@jsdoc - * Adds avatars to the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list is + * Adds avatars to the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list is * played. * @function Audio.addToSoloList * @param {Uuid[]} ids - Avatar IDs to add to the solo list. @@ -54,25 +62,25 @@ class AudioScriptingInterface : public QObject, public Dependency { * // Find nearby avatars. * var RANGE = 100; // m * var nearbyAvatars = AvatarList.getAvatarsInRange(MyAvatar.position, RANGE); - * + * * // Remove own avatar from list. * var myAvatarIndex = nearbyAvatars.indexOf(MyAvatar.sessionUUID); * if (myAvatarIndex !== -1) { * nearbyAvatars.splice(myAvatarIndex, 1); * } - * + * * if (nearbyAvatars.length > 0) { * // Listen to only one of the nearby avatars. * var avatarName = AvatarList.getAvatar(nearbyAvatars[0]).displayName; * print("Listening only to " + avatarName); * Audio.addToSoloList([nearbyAvatars[0]]); - * + * * // Stop listening to only the one avatar after a short while. * Script.setTimeout(function () { * print("Finished listening only to " + avatarName); * Audio.resetSoloList(); * }, 10000); // 10s - * + * * } else { * print("No nearby avatars"); * } @@ -82,7 +90,7 @@ class AudioScriptingInterface : public QObject, public Dependency { } /*@jsdoc - * Removes avatars from the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list + * Removes avatars from the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list * is played. * @function Audio.removeFromSoloList * @param {Uuid[]} ids - Avatar IDs to remove from the solo list. @@ -100,44 +108,44 @@ class AudioScriptingInterface : public QObject, public Dependency { } /*@jsdoc - * Gets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed only + * Gets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed only * if you're unmuted or are using push-to-talk. * @function Audio.getServerEcho - * @returns {boolean} true if echoing microphone audio back to you from the server is enabled, + * @returns {boolean} true if echoing microphone audio back to you from the server is enabled, * false if it isn't. */ Q_INVOKABLE bool getServerEcho(); /*@jsdoc - * Sets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed + * Sets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed * only if you're unmuted or are using push-to-talk. * @function Audio.setServerEcho - * @param {boolean} serverEcho - true to enable echoing microphone back to you from the server, + * @param {boolean} serverEcho - true to enable echoing microphone back to you from the server, * false to disable. */ Q_INVOKABLE void setServerEcho(bool serverEcho); /*@jsdoc - * Toggles the echoing of microphone audio back to you from the server. When enabled, microphone audio is echoed only if + * Toggles the echoing of microphone audio back to you from the server. When enabled, microphone audio is echoed only if * you're unmuted or are using push-to-talk. * @function Audio.toggleServerEcho */ Q_INVOKABLE void toggleServerEcho(); /*@jsdoc - * Gets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed + * Gets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed * even if you're muted or not using push-to-talk. * @function Audio.getLocalEcho - * @returns {boolean} true if echoing microphone audio back to you from the client is enabled, + * @returns {boolean} true if echoing microphone audio back to you from the client is enabled, * false if it isn't. */ Q_INVOKABLE bool getLocalEcho(); /*@jsdoc - * Sets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed + * Sets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed * even if you're muted or not using push-to-talk. * @function Audio.setLocalEcho - * @parm {boolean} localEcho - true to enable echoing microphone audio back to you from the client, + * @parm {boolean} localEcho - true to enable echoing microphone audio back to you from the client, * false to disable. * @example Echo local audio for a few seconds. * Audio.setLocalEcho(true); @@ -148,7 +156,7 @@ class AudioScriptingInterface : public QObject, public Dependency { Q_INVOKABLE void setLocalEcho(bool localEcho); /*@jsdoc - * Toggles the echoing of microphone audio back to you by the client. When enabled, microphone audio is echoed even if + * Toggles the echoing of microphone audio back to you by the client. When enabled, microphone audio is echoed even if * you're muted or not using push-to-talk. * @function Audio.toggleLocalEcho */ @@ -161,32 +169,32 @@ class AudioScriptingInterface : public QObject, public Dependency { // these methods are protected to stop C++ callers from calling, but invokable from script /*@jsdoc - * Starts playing or "injecting" the content of an audio file. The sound is played globally (sent to the audio - * mixer) so that everyone hears it, unless the injectorOptions has localOnly set to - * true in which case only the client hears the sound played. No sound is played if sent to the audio mixer - * but the client is not connected to an audio mixer. The {@link AudioInjector} object returned by the function can be used + * Starts playing or "injecting" the content of an audio file. The sound is played globally (sent to the audio + * mixer) so that everyone hears it, unless the injectorOptions has localOnly set to + * true in which case only the client hears the sound played. No sound is played if sent to the audio mixer + * but the client is not connected to an audio mixer. The {@link AudioInjector} object returned by the function can be used * to control the playback and get information about its current state. * @function Audio.playSound - * @param {SoundObject} sound - The content of an audio file, loaded using {@link SoundCache.getSound}. See + * @param {SoundObject} sound - The content of an audio file, loaded using {@link SoundCache.getSound}. See * {@link SoundObject} for supported formats. - * @param {AudioInjector.AudioInjectorOptions} [injectorOptions={}] - Configures where and how the audio injector plays the + * @param {AudioInjector.AudioInjectorOptions} [injectorOptions={}] - Configures where and how the audio injector plays the * audio file. * @returns {AudioInjector} The audio injector that plays the audio file. * @example Play a sound. * var sound = SoundCache.getSound("https://cdn-1.vircadia.com/us-c-1/ken/samples/forest_ambiX.wav"); - * + * * function playSound() { * var injectorOptions = { * position: MyAvatar.position * }; * var injector = Audio.playSound(sound, injectorOptions); * } - * + * * function onSoundReady() { * sound.ready.disconnect(onSoundReady); * playSound(); * } - * + * * if (sound.downloaded) { * playSound(); * } else { @@ -196,18 +204,18 @@ class AudioScriptingInterface : public QObject, public Dependency { Q_INVOKABLE ScriptAudioInjector* playSound(SharedSoundPointer sound, const AudioInjectorOptions& injectorOptions = AudioInjectorOptions()); /*@jsdoc - * Starts playing the content of an audio file locally (isn't sent to the audio mixer). This is the same as calling - * {@link Audio.playSound} with {@link AudioInjector.AudioInjectorOptions} localOnly set true and + * Starts playing the content of an audio file locally (isn't sent to the audio mixer). This is the same as calling + * {@link Audio.playSound} with {@link AudioInjector.AudioInjectorOptions} localOnly set true and * the specified position. * @function Audio.playSystemSound - * @param {SoundObject} sound - The content of an audio file, which is loaded using {@link SoundCache.getSound}. See + * @param {SoundObject} sound - The content of an audio file, which is loaded using {@link SoundCache.getSound}. See * {@link SoundObject} for supported formats. * @returns {AudioInjector} The audio injector that plays the audio file. */ Q_INVOKABLE ScriptAudioInjector* playSystemSound(SharedSoundPointer sound); /*@jsdoc - * Sets whether the audio input should be used in stereo. If the audio input doesn't support stereo then setting a value + * Sets whether the audio input should be used in stereo. If the audio input doesn't support stereo then setting a value * of true has no effect. * @function Audio.setStereoInput * @param {boolean} stereo - true if the audio input should be used in stereo, otherwise false. @@ -217,57 +225,57 @@ class AudioScriptingInterface : public QObject, public Dependency { /*@jsdoc * Gets whether the audio input is used in stereo. * @function Audio.isStereoInput - * @returns {boolean} true if the audio input is used in stereo, otherwise false. + * @returns {boolean} true if the audio input is used in stereo, otherwise false. */ Q_INVOKABLE bool isStereoInput(); signals: /*@jsdoc - * Triggered when the client is muted by the mixer because their loudness value for the noise background has reached the + * Triggered when the client is muted by the mixer because their loudness value for the noise background has reached the * threshold set for the domain (in the server settings). * @function Audio.mutedByMixer - * @returns {Signal} + * @returns {Signal} */ void mutedByMixer(); /*@jsdoc - * Triggered when the client is muted by the mixer because they're within a certain radius (50m) of someone who requested + * Triggered when the client is muted by the mixer because they're within a certain radius (50m) of someone who requested * the mute through Developer > Audio > Mute Environment. * @function Audio.environmentMuted - * @returns {Signal} + * @returns {Signal} */ void environmentMuted(); /*@jsdoc * Triggered when the client receives its first packet from the audio mixer. * @function Audio.receivedFirstPacket - * @returns {Signal} + * @returns {Signal} */ void receivedFirstPacket(); /*@jsdoc * Triggered when the client is disconnected from the audio mixer. * @function Audio.disconnected - * @returns {Signal} + * @returns {Signal} */ void disconnected(); /*@jsdoc - * Triggered when the noise gate is opened. The input audio signal is no longer blocked (fully attenuated) because it has - * risen above an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is + * Triggered when the noise gate is opened. The input audio signal is no longer blocked (fully attenuated) because it has + * risen above an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is * true. * @function Audio.noiseGateOpened - * @returns {Signal} + * @returns {Signal} */ void noiseGateOpened(); /*@jsdoc - * Triggered when the noise gate is closed. The input audio signal is blocked (fully attenuated) because it has fallen - * below an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is + * Triggered when the noise gate is closed. The input audio signal is blocked (fully attenuated) because it has fallen + * below an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is * true. * @function Audio.noiseGateClosed - * @returns {Signal} + * @returns {Signal} */ void noiseGateClosed(); @@ -275,7 +283,7 @@ class AudioScriptingInterface : public QObject, public Dependency { * Triggered when a frame of audio input is processed. * @function Audio.inputReceived * @param {Int16Array} inputSamples - The audio input processed. - * @returns {Signal} + * @returns {Signal} */ void inputReceived(const QByteArray& inputSamples); diff --git a/libraries/shared-gui/src/AABox.cpp b/libraries/shared-gui/src/AABox.cpp index 2040340a11c..ca438b4496c 100644 --- a/libraries/shared-gui/src/AABox.cpp +++ b/libraries/shared-gui/src/AABox.cpp @@ -20,102 +20,101 @@ const glm::vec3 AABox::INFINITY_VECTOR(std::numeric_limits::infinity()); AABox::AABox(const AACube& other) : - _corner(other.getCorner()), _scale(other.getScale(), other.getScale(), other.getScale()) { + _data{other.getCorner(), {other.getScale(), other.getScale(), other.getScale()}} { } AABox::AABox(const Extents& other) : - _corner(other.minimum), - _scale(other.maximum - other.minimum) { + _data{other.minimum, other.maximum - other.minimum} { } AABox::AABox(const glm::vec3& corner, float size) : - _corner(corner), _scale(size, size, size) { + _data{corner, {size, size, size}} { }; AABox::AABox(const glm::vec3& corner, const glm::vec3& dimensions) : - _corner(corner), _scale(dimensions) { + _data{corner, dimensions} { }; -AABox::AABox() : _corner(INFINITY_VECTOR), _scale(0.0f) { +AABox::AABox() : _data{INFINITY_VECTOR, Vectors::ZERO} { }; glm::vec3 AABox::calcCenter() const { - glm::vec3 center(_corner); - center += (_scale * 0.5f); + glm::vec3 center(_data.corner); + center += (_data.scale * 0.5f); return center; } glm::vec3 AABox::getVertex(BoxVertex vertex) const { switch (vertex) { case BOTTOM_LEFT_NEAR: - return _corner + glm::vec3(_scale.x, 0, 0); + return _data.corner + glm::vec3(_data.scale.x, 0, 0); case BOTTOM_RIGHT_NEAR: - return _corner; + return _data.corner; case TOP_RIGHT_NEAR: - return _corner + glm::vec3(0, _scale.y, 0); + return _data.corner + glm::vec3(0, _data.scale.y, 0); case TOP_LEFT_NEAR: - return _corner + glm::vec3(_scale.x, _scale.y, 0); + return _data.corner + glm::vec3(_data.scale.x, _data.scale.y, 0); case BOTTOM_LEFT_FAR: - return _corner + glm::vec3(_scale.x, 0, _scale.z); + return _data.corner + glm::vec3(_data.scale.x, 0, _data.scale.z); case BOTTOM_RIGHT_FAR: - return _corner + glm::vec3(0, 0, _scale.z); + return _data.corner + glm::vec3(0, 0, _data.scale.z); case TOP_RIGHT_FAR: - return _corner + glm::vec3(0, _scale.y, _scale.z); + return _data.corner + glm::vec3(0, _data.scale.y, _data.scale.z); default: //quiet windows warnings case TOP_LEFT_FAR: - return _corner + _scale; + return _data.corner + _data.scale; } } void AABox::setBox(const glm::vec3& corner, float scale) { - _corner = corner; - _scale = glm::vec3(scale, scale, scale); + _data.corner = corner; + _data.scale = glm::vec3(scale, scale, scale); } void AABox::setBox(const glm::vec3& corner, const glm::vec3& scale) { - _corner = corner; - _scale = scale; + _data.corner = corner; + _data.scale = scale; } glm::vec3 AABox::getFarthestVertex(const glm::vec3& normal) const { - glm::vec3 result = _corner; + glm::vec3 result = _data.corner; // This is a branchless version of: //if (normal.x > 0.0f) { - // result.x += _scale.x; + // result.x += _data.scale.x; //} //if (normal.y > 0.0f) { - // result.y += _scale.y; + // result.y += _data.scale.y; //} //if (normal.z > 0.0f) { - // result.z += _scale.z; + // result.z += _data.scale.z; //} float blend = (float)(normal.x > 0.0f); - result.x += blend * _scale.x + (1.0f - blend) * 0.0f; + result.x += blend * _data.scale.x + (1.0f - blend) * 0.0f; blend = (float)(normal.y > 0.0f); - result.y += blend * _scale.y + (1.0f - blend) * 0.0f; + result.y += blend * _data.scale.y + (1.0f - blend) * 0.0f; blend = (float)(normal.z > 0.0f); - result.z += blend * _scale.z + (1.0f - blend) * 0.0f; + result.z += blend * _data.scale.z + (1.0f - blend) * 0.0f; return result; } glm::vec3 AABox::getNearestVertex(const glm::vec3& normal) const { - glm::vec3 result = _corner; + glm::vec3 result = _data.corner; // This is a branchless version of: //if (normal.x < 0.0f) { - // result.x += _scale.x; + // result.x += _data.scale.x; //} //if (normal.y < 0.0f) { - // result.y += _scale.y; + // result.y += _data.scale.y; //} //if (normal.z < 0.0f) { - // result.z += _scale.z; + // result.z += _data.scale.z; //} float blend = (float)(normal.x < 0.0f); - result.x += blend * _scale.x + (1.0f - blend) * 0.0f; + result.x += blend * _data.scale.x + (1.0f - blend) * 0.0f; blend = (float)(normal.y < 0.0f); - result.y += blend * _scale.y + (1.0f - blend) * 0.0f; + result.y += blend * _data.scale.y + (1.0f - blend) * 0.0f; blend = (float)(normal.z < 0.0f); - result.z += blend * _scale.z + (1.0f - blend) * 0.0f; + result.z += blend * _data.scale.z + (1.0f - blend) * 0.0f; return result; } @@ -124,7 +123,7 @@ bool AABox::contains(const Triangle& triangle) const { } bool AABox::contains(const glm::vec3& point) const { - return aaBoxContains(point, _corner, _scale); + return aaBoxContains(point, _data.corner, _data.scale); } bool AABox::contains(const AABox& otherBox) const { @@ -138,13 +137,7 @@ bool AABox::contains(const AABox& otherBox) const { } bool AABox::touches(const AABox& otherBox) const { - glm::vec3 relativeCenter = _corner - otherBox._corner + ((_scale - otherBox._scale) * 0.5f); - - glm::vec3 totalHalfScale = (_scale + otherBox._scale) * 0.5f; - - return fabsf(relativeCenter.x) <= totalHalfScale.x && - fabsf(relativeCenter.y) <= totalHalfScale.y && - fabsf(relativeCenter.z) <= totalHalfScale.z; + return isTouching(_data, otherBox._data); } bool AABox::contains(const AACube& otherCube) const { @@ -158,9 +151,9 @@ bool AABox::contains(const AACube& otherCube) const { } bool AABox::touches(const AACube& otherCube) const { - glm::vec3 relativeCenter = _corner - otherCube.getCorner() + ((_scale - otherCube.getDimensions()) * 0.5f); + glm::vec3 relativeCenter = _data.corner - otherCube.getCorner() + ((_data.scale - otherCube.getDimensions()) * 0.5f); - glm::vec3 totalHalfScale = (_scale + otherCube.getDimensions()) * 0.5f; + glm::vec3 totalHalfScale = (_data.scale + otherCube.getDimensions()) * 0.5f; return fabsf(relativeCenter.x) <= totalHalfScale.x && fabsf(relativeCenter.y) <= totalHalfScale.y && @@ -173,9 +166,9 @@ static bool isWithinExpanded(float value, float corner, float size, float expans } bool AABox::expandedContains(const glm::vec3& point, float expansion) const { - return isWithinExpanded(point.x, _corner.x, _scale.x, expansion) && - isWithinExpanded(point.y, _corner.y, _scale.y, expansion) && - isWithinExpanded(point.z, _corner.z, _scale.z, expansion); + return isWithinExpanded(point.x, _data.corner.x, _data.scale.x, expansion) && + isWithinExpanded(point.y, _data.corner.y, _data.scale.y, expansion) && + isWithinExpanded(point.z, _data.corner.z, _data.scale.z, expansion); } bool AABox::expandedIntersectsSegment(const glm::vec3& start, const glm::vec3& end, float expansion) const { @@ -184,8 +177,8 @@ bool AABox::expandedIntersectsSegment(const glm::vec3& start, const glm::vec3& e return true; } // check each axis - glm::vec3 expandedCorner = _corner - glm::vec3(expansion, expansion, expansion); - glm::vec3 expandedSize = _scale + glm::vec3(expansion, expansion, expansion) * 2.0f; + glm::vec3 expandedCorner = _data.corner - glm::vec3(expansion, expansion, expansion); + glm::vec3 expandedSize = _data.scale + glm::vec3(expansion, expansion, expansion) * 2.0f; glm::vec3 direction = end - start; float axisDistance; return (findIntersection(start.x, direction.x, expandedCorner.x, expandedSize.x, axisDistance) && @@ -204,19 +197,19 @@ bool AABox::expandedIntersectsSegment(const glm::vec3& start, const glm::vec3& e bool AABox::findRayIntersection(const glm::vec3& origin, const glm::vec3& direction, const glm::vec3& invDirection, float& distance, BoxFace& face, glm::vec3& surfaceNormal) const { - return findRayAABoxIntersection(origin, direction, invDirection, _corner, _scale, distance, face, surfaceNormal); + return findRayAABoxIntersection(origin, direction, invDirection, _data.corner, _data.scale, distance, face, surfaceNormal); } bool AABox::findParabolaIntersection(const glm::vec3& origin, const glm::vec3& velocity, const glm::vec3& acceleration, float& parabolicDistance, BoxFace& face, glm::vec3& surfaceNormal) const { - return findParabolaAABoxIntersection(origin, velocity, acceleration, _corner, _scale, parabolicDistance, face, surfaceNormal); + return findParabolaAABoxIntersection(origin, velocity, acceleration, _data.corner, _data.scale, parabolicDistance, face, surfaceNormal); } bool AABox::rayHitsBoundingSphere(const glm::vec3& origin, const glm::vec3& direction) const { glm::vec3 localCenter = calcCenter() - origin; float distance = glm::dot(localCenter, direction); const float ONE_OVER_TWO_SQUARED = 0.25f; - float radiusSquared = ONE_OVER_TWO_SQUARED * glm::length2(_scale); + float radiusSquared = ONE_OVER_TWO_SQUARED * glm::length2(_data.scale); return (glm::length2(localCenter) < radiusSquared || (glm::abs(distance) > 0.0f && glm::distance2(distance * direction, localCenter) < radiusSquared)); } @@ -224,7 +217,7 @@ bool AABox::rayHitsBoundingSphere(const glm::vec3& origin, const glm::vec3& dire bool AABox::parabolaPlaneIntersectsBoundingSphere(const glm::vec3& origin, const glm::vec3& velocity, const glm::vec3& acceleration, const glm::vec3& normal) const { glm::vec3 localCenter = calcCenter() - origin; const float ONE_OVER_TWO_SQUARED = 0.25f; - float radiusSquared = ONE_OVER_TWO_SQUARED * glm::length2(_scale); + float radiusSquared = ONE_OVER_TWO_SQUARED * glm::length2(_data.scale); // origin is inside the sphere if (glm::length2(localCenter) < radiusSquared) { @@ -246,7 +239,7 @@ bool AABox::parabolaPlaneIntersectsBoundingSphere(const glm::vec3& origin, const bool AABox::touchesSphere(const glm::vec3& center, float radius) const { // Avro's algorithm from this paper: http://www.mrtc.mdh.se/projects/3Dgraphics/paperF.pdf - glm::vec3 e = glm::max(_corner - center, Vectors::ZERO) + glm::max(center - _corner - _scale, Vectors::ZERO); + glm::vec3 e = glm::max(_data.corner - center, Vectors::ZERO) + glm::max(center - _data.corner - _data.scale, Vectors::ZERO); return glm::length2(e) <= radius * radius; } @@ -304,29 +297,29 @@ bool AABox::findCapsulePenetration(const glm::vec3& start, const glm::vec3& end, glm::vec3 AABox::getClosestPointOnFace(const glm::vec3& point, BoxFace face) const { switch (face) { case MIN_X_FACE: - return glm::clamp(point, glm::vec3(_corner.x, _corner.y, _corner.z), - glm::vec3(_corner.x, _corner.y + _scale.y, _corner.z + _scale.z)); + return glm::clamp(point, glm::vec3(_data.corner.x, _data.corner.y, _data.corner.z), + glm::vec3(_data.corner.x, _data.corner.y + _data.scale.y, _data.corner.z + _data.scale.z)); case MAX_X_FACE: - return glm::clamp(point, glm::vec3(_corner.x + _scale.x, _corner.y, _corner.z), - glm::vec3(_corner.x + _scale.x, _corner.y + _scale.y, _corner.z + _scale.z)); + return glm::clamp(point, glm::vec3(_data.corner.x + _data.scale.x, _data.corner.y, _data.corner.z), + glm::vec3(_data.corner.x + _data.scale.x, _data.corner.y + _data.scale.y, _data.corner.z + _data.scale.z)); case MIN_Y_FACE: - return glm::clamp(point, glm::vec3(_corner.x, _corner.y, _corner.z), - glm::vec3(_corner.x + _scale.x, _corner.y, _corner.z + _scale.z)); + return glm::clamp(point, glm::vec3(_data.corner.x, _data.corner.y, _data.corner.z), + glm::vec3(_data.corner.x + _data.scale.x, _data.corner.y, _data.corner.z + _data.scale.z)); case MAX_Y_FACE: - return glm::clamp(point, glm::vec3(_corner.x, _corner.y + _scale.y, _corner.z), - glm::vec3(_corner.x + _scale.x, _corner.y + _scale.y, _corner.z + _scale.z)); + return glm::clamp(point, glm::vec3(_data.corner.x, _data.corner.y + _data.scale.y, _data.corner.z), + glm::vec3(_data.corner.x + _data.scale.x, _data.corner.y + _data.scale.y, _data.corner.z + _data.scale.z)); case MIN_Z_FACE: - return glm::clamp(point, glm::vec3(_corner.x, _corner.y, _corner.z), - glm::vec3(_corner.x + _scale.x, _corner.y + _scale.y, _corner.z)); + return glm::clamp(point, glm::vec3(_data.corner.x, _data.corner.y, _data.corner.z), + glm::vec3(_data.corner.x + _data.scale.x, _data.corner.y + _data.scale.y, _data.corner.z)); default: //quiet windows warnings case MAX_Z_FACE: - return glm::clamp(point, glm::vec3(_corner.x, _corner.y, _corner.z + _scale.z), - glm::vec3(_corner.x + _scale.x, _corner.y + _scale.y, _corner.z + _scale.z)); + return glm::clamp(point, glm::vec3(_data.corner.x, _data.corner.y, _data.corner.z + _data.scale.z), + glm::vec3(_data.corner.x + _data.scale.x, _data.corner.y + _data.scale.y, _data.corner.z + _data.scale.z)); } } @@ -376,7 +369,7 @@ glm::vec3 AABox::getClosestPointOnFace(const glm::vec4& origin, const glm::vec4& glm::vec4 thirdAxisMaxPlane = getPlane((BoxFace)(thirdAxis * 2 + 1)); glm::vec4 offset = glm::vec4(0.0f, 0.0f, 0.0f, - glm::dot(glm::vec3(secondAxisMaxPlane + thirdAxisMaxPlane), _scale) * 0.5f); + glm::dot(glm::vec3(secondAxisMaxPlane + thirdAxisMaxPlane), _data.scale) * 0.5f); glm::vec4 diagonals[] = { secondAxisMinPlane + thirdAxisMaxPlane + offset, secondAxisMaxPlane + thirdAxisMaxPlane + offset }; @@ -399,12 +392,12 @@ glm::vec3 AABox::getClosestPointOnFace(const glm::vec4& origin, const glm::vec4& bool AABox::touchesAAEllipsoid(const glm::vec3& center, const glm::vec3& radials) const { // handle case where ellipsoid's alix-aligned box doesn't touch this AABox - if (_corner.x - radials.x > center.x || - _corner.y - radials.y > center.y || - _corner.z - radials.z > center.z || - _corner.x + _scale.x + radials.x < center.x || - _corner.y + _scale.y + radials.y < center.y || - _corner.z + _scale.z + radials.z < center.z) { + if (_data.corner.x - radials.x > center.x || + _data.corner.y - radials.y > center.y || + _data.corner.z - radials.z > center.z || + _data.corner.x + _data.scale.x + radials.x < center.x || + _data.corner.y + _data.scale.y + radials.y < center.y || + _data.corner.z + _data.scale.z + radials.z < center.z) { return false; } @@ -431,13 +424,13 @@ bool AABox::touchesAAEllipsoid(const glm::vec3& center, const glm::vec3& radials glm::vec4 AABox::getPlane(BoxFace face) const { switch (face) { - case MIN_X_FACE: return glm::vec4(-1.0f, 0.0f, 0.0f, _corner.x); - case MAX_X_FACE: return glm::vec4(1.0f, 0.0f, 0.0f, -_corner.x - _scale.x); - case MIN_Y_FACE: return glm::vec4(0.0f, -1.0f, 0.0f, _corner.y); - case MAX_Y_FACE: return glm::vec4(0.0f, 1.0f, 0.0f, -_corner.y - _scale.y); - case MIN_Z_FACE: return glm::vec4(0.0f, 0.0f, -1.0f, _corner.z); + case MIN_X_FACE: return glm::vec4(-1.0f, 0.0f, 0.0f, _data.corner.x); + case MAX_X_FACE: return glm::vec4(1.0f, 0.0f, 0.0f, -_data.corner.x - _data.scale.x); + case MIN_Y_FACE: return glm::vec4(0.0f, -1.0f, 0.0f, _data.corner.y); + case MAX_Y_FACE: return glm::vec4(0.0f, 1.0f, 0.0f, -_data.corner.y - _data.scale.y); + case MIN_Z_FACE: return glm::vec4(0.0f, 0.0f, -1.0f, _data.corner.z); default: //quiet windows warnings - case MAX_Z_FACE: return glm::vec4(0.0f, 0.0f, 1.0f, -_corner.z - _scale.z); + case MAX_Z_FACE: return glm::vec4(0.0f, 0.0f, 1.0f, -_data.corner.z - _data.scale.z); } } @@ -454,7 +447,7 @@ BoxFace AABox::getOppositeFace(BoxFace face) { } AABox AABox::clamp(const glm::vec3& min, const glm::vec3& max) const { - glm::vec3 clampedCorner = glm::clamp(_corner, min, max); + glm::vec3 clampedCorner = glm::clamp(_data.corner, min, max); glm::vec3 clampedTopFarLeft = glm::clamp(calcTopFarLeft(), min, max); glm::vec3 clampedScale = clampedTopFarLeft - clampedCorner; @@ -462,7 +455,7 @@ AABox AABox::clamp(const glm::vec3& min, const glm::vec3& max) const { } AABox AABox::clamp(float min, float max) const { - glm::vec3 clampedCorner = glm::clamp(_corner, min, max); + glm::vec3 clampedCorner = glm::clamp(_data.corner, min, max); glm::vec3 clampedTopFarLeft = glm::clamp(calcTopFarLeft(), min, max); glm::vec3 clampedScale = clampedTopFarLeft - clampedCorner; @@ -470,33 +463,33 @@ AABox AABox::clamp(float min, float max) const { } void AABox::embiggen(float scale) { - _corner += scale * (-0.5f * _scale); - _scale *= scale; + _data.corner += scale * (-0.5f * _data.scale); + _data.scale *= scale; } void AABox::embiggen(const glm::vec3& scale) { - _corner += scale * (-0.5f * _scale); - _scale *= scale; + _data.corner += scale * (-0.5f * _data.scale); + _data.scale *= scale; } void AABox::setScaleStayCentered(const glm::vec3& scale) { - _corner -= 0.5f * (scale - _scale); - _scale = scale; + _data.corner -= 0.5f * (scale - _data.scale); + _data.scale = scale; } void AABox::scale(float scale) { - _corner *= scale; - _scale *= scale; + _data.corner *= scale; + _data.scale *= scale; } void AABox::scale(const glm::vec3& scale) { - _corner *= scale; - _scale *= scale; + _data.corner *= scale; + _data.scale *= scale; } void AABox::rotate(const glm::quat& rotation) { - auto minimum = _corner; - auto maximum = _corner + _scale; + auto minimum = _data.corner; + auto maximum = _data.corner + _data.scale; glm::vec3 bottomLeftNear(minimum.x, minimum.y, minimum.z); glm::vec3 bottomRightNear(maximum.x, minimum.y, minimum.z); @@ -534,8 +527,8 @@ void AABox::rotate(const glm::quat& rotation) { glm::max(topLeftFarRotated, topRightFarRotated))))))); - _corner = minimum; - _scale = maximum - minimum; + _data.corner = minimum; + _data.scale = maximum - minimum; } void AABox::transform(const Transform& transform) { @@ -547,8 +540,8 @@ void AABox::transform(const Transform& transform) { // Logic based on http://clb.demon.fi/MathGeoLib/nightly/docs/AABB.cpp_code.html#471 void AABox::transform(const glm::mat4& matrix) { // FIXME use simd operations - auto halfSize = _scale * 0.5f; - auto center = _corner + halfSize; + auto halfSize = _data.scale * 0.5f; + auto center = _data.corner + halfSize; halfSize = abs(halfSize); auto mm = glm::transpose(glm::mat3(matrix)); vec3 newDir = vec3( @@ -558,43 +551,43 @@ void AABox::transform(const glm::mat4& matrix) { ); auto newCenter = transformPoint(matrix, center); - _corner = newCenter - newDir; - _scale = newDir * 2.0f; + _data.corner = newCenter - newDir; + _data.scale = newDir * 2.0f; } AABox AABox::getOctreeChild(OctreeChild child) const { AABox result(*this); // self switch (child) { case topLeftNear: - result._corner.y += _scale.y / 2.0f; + result._data.corner.y += _data.scale.y / 2.0f; break; case topLeftFar: - result._corner.y += _scale.y / 2.0f; - result._corner.z += _scale.z / 2.0f; + result._data.corner.y += _data.scale.y / 2.0f; + result._data.corner.z += _data.scale.z / 2.0f; break; case topRightNear: - result._corner.y += _scale.y / 2.0f; - result._corner.x += _scale.x / 2.0f; + result._data.corner.y += _data.scale.y / 2.0f; + result._data.corner.x += _data.scale.x / 2.0f; break; case topRightFar: - result._corner.y += _scale.y / 2.0f; - result._corner.x += _scale.x / 2.0f; - result._corner.z += _scale.z / 2.0f; + result._data.corner.y += _data.scale.y / 2.0f; + result._data.corner.x += _data.scale.x / 2.0f; + result._data.corner.z += _data.scale.z / 2.0f; break; case bottomLeftNear: - // _corner = same as parent + // _data.corner = same as parent break; case bottomLeftFar: - result._corner.z += _scale.z / 2.0f; + result._data.corner.z += _data.scale.z / 2.0f; break; case bottomRightNear: - result._corner.x += _scale.x / 2.0f; + result._data.corner.x += _data.scale.x / 2.0f; break; case bottomRightFar: - result._corner.x += _scale.x / 2.0f; - result._corner.z += _scale.z / 2.0f; + result._data.corner.x += _data.scale.x / 2.0f; + result._data.corner.z += _data.scale.z / 2.0f; break; } - result._scale /= 2.0f; // everything is half the scale + result._data.scale /= 2.0f; // everything is half the scale return result; } diff --git a/libraries/shared-gui/src/AABox.h b/libraries/shared-gui/src/AABox.h index c5fb85fdd11..69fe4a01a9c 100644 --- a/libraries/shared-gui/src/AABox.h +++ b/libraries/shared-gui/src/AABox.h @@ -19,6 +19,8 @@ #include +#include + #include "BoxBase.h" #include "GeometryUtil.h" #include "StreamUtils.h" @@ -43,20 +45,20 @@ class AABox { glm::vec3 getFarthestVertex(const glm::vec3& normal) const; // return vertex most parallel to normal glm::vec3 getNearestVertex(const glm::vec3& normal) const; // return vertex most anti-parallel to normal - const glm::vec3& getCorner() const { return _corner; } - const glm::vec3& getScale() const { return _scale; } - const glm::vec3& getDimensions() const { return _scale; } - float getLargestDimension() const { return glm::max(_scale.x, glm::max(_scale.y, _scale.z)); } + const glm::vec3& getCorner() const { return _data.corner; } + const glm::vec3& getScale() const { return _data.scale; } + const glm::vec3& getDimensions() const { return _data.scale; } + float getLargestDimension() const { return glm::max(_data.scale.x, glm::max(_data.scale.y, _data.scale.z)); } glm::vec3 calcCenter() const; - glm::vec3 calcTopFarLeft() const { return _corner + _scale; } + glm::vec3 calcTopFarLeft() const { return _data.corner + _data.scale; } - const glm::vec3& getMinimum() const { return _corner; } - glm::vec3 getMaximum() const { return _corner + _scale; } + const glm::vec3& getMinimum() const { return _data.corner; } + glm::vec3 getMaximum() const { return _data.corner + _data.scale; } glm::vec3 getVertex(BoxVertex vertex) const; - const glm::vec3& getMinimumPoint() const { return _corner; } + const glm::vec3& getMinimumPoint() const { return _data.corner; } glm::vec3 getMaximumPoint() const { return calcTopFarLeft(); } bool contains(const Triangle& triangle) const; @@ -80,31 +82,31 @@ class AABox { bool findSpherePenetration(const glm::vec3& center, float radius, glm::vec3& penetration) const; bool findCapsulePenetration(const glm::vec3& start, const glm::vec3& end, float radius, glm::vec3& penetration) const; - bool isNull() const { return _scale == glm::vec3(0.0f, 0.0f, 0.0f); } + bool isNull() const { return _data.scale == glm::vec3(0.0f, 0.0f, 0.0f); } AABox clamp(const glm::vec3& min, const glm::vec3& max) const; AABox clamp(float min, float max) const; inline AABox& operator+=(const glm::vec3& point) { bool valid = !isInvalid(); - glm::vec3 maximum = glm::max(_corner + _scale, point); - _corner = glm::min(_corner, point); + glm::vec3 maximum = glm::max(_data.corner + _data.scale, point); + _data.corner = glm::min(_data.corner, point); if (valid) { - _scale = maximum - _corner; + _data.scale = maximum - _data.corner; } return (*this); } inline AABox& operator+=(const AABox& box) { if (!box.isInvalid()) { - (*this) += box._corner; + (*this) += box._data.corner; (*this) += box.calcTopFarLeft(); } return (*this); } // Translate the AABox just moving the corner - void translate(const glm::vec3& translation) { _corner += translation; } + void translate(const glm::vec3& translation) { _data.corner += translation; } // Rotate the AABox around its frame origin // meaning rotating the corners of the AABox around the point {0,0,0} and reevaluating the min max @@ -129,9 +131,9 @@ class AABox { static const glm::vec3 INFINITY_VECTOR; - bool isInvalid() const { return _corner.x == std::numeric_limits::infinity(); } + bool isInvalid() const { return _data.corner.x == std::numeric_limits::infinity(); } - void clear() { _corner = INFINITY_VECTOR; _scale = glm::vec3(0.0f); } + void clear() { _data.corner = INFINITY_VECTOR; _data.scale = glm::vec3(0.0f); } typedef enum { topLeftNear, @@ -157,8 +159,7 @@ class AABox { void checkPossibleParabolicIntersection(float t, int i, float& minDistance, const glm::vec3& origin, const glm::vec3& velocity, const glm::vec3& acceleration, bool& hit) const; - glm::vec3 _corner; - glm::vec3 _scale; + AABoxData _data; }; inline bool operator==(const AABox& a, const AABox& b) { diff --git a/libraries/shared/src/AABoxData.h b/libraries/shared/src/AABoxData.h new file mode 100644 index 00000000000..8f728f70cbf --- /dev/null +++ b/libraries/shared/src/AABoxData.h @@ -0,0 +1,34 @@ +// +// AABoxData.h +// libraries/shared/src +// +// Created by Nshan G. on 3 July 2022 +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + + +#ifndef LIBRARIES_SHARED_SRC_AABOXDARA_H +#define LIBRARIES_SHARED_SRC_AABOXDARA_H + +#include + +struct AABoxData { + glm::vec3 corner; + glm::vec3 scale; +}; + +inline bool isTouching(AABoxData one, AABoxData other) { + glm::vec3 relativeCenter = one.corner - other.corner + ((one.scale - other.scale) * 0.5f); + + glm::vec3 totalHalfScale = (one.scale + other.scale) * 0.5f; + + return fabsf(relativeCenter.x) <= totalHalfScale.x && + fabsf(relativeCenter.y) <= totalHalfScale.y && + fabsf(relativeCenter.z) <= totalHalfScale.z; +} + +#endif /* end of include guard */ diff --git a/libraries/shared/src/Codec.h b/libraries/shared/src/Codec.h new file mode 100644 index 00000000000..6ffb6f2fef4 --- /dev/null +++ b/libraries/shared/src/Codec.h @@ -0,0 +1,41 @@ +// +// Codec.h +// libraries/shared/src +// +// Created by Nshan G. 17 June 2022 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef VIRCADIA_LIBRARIES_SHARED_SRC_CODEC_H +#define VIRCADIA_LIBRARIES_SHARED_SRC_CODEC_H + +#include +#include + +class Encoder { +public: + virtual ~Encoder() { } + virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; +}; + +class Decoder { +public: + virtual ~Decoder() { } + virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) = 0; + + virtual void lostFrame(QByteArray& decodedBuffer) = 0; +}; + +class Codec { +public: + virtual const QString getName() const = 0; + virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; + virtual Decoder* createDecoder(int sampleRate, int numChannels) = 0; + virtual void releaseEncoder(Encoder* encoder) = 0; + virtual void releaseDecoder(Decoder* decoder) = 0; +}; + +#endif /* end of include guard */ diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index 9f635b11a1d..2e94e05c348 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -16,30 +16,22 @@ #include #endif -// WEBRTC_AUDIO: WebRTC audio features, e.g., echo canceling. +// WEBRTC_AUDIO: WebRTC audio features, e.g., echo canceling (defined in cmake). // WEBRTC_DATA_CHANNELS: WebRTC client-server connections in parallel with UDP (defined in cmake). #if defined(Q_OS_MAC) -# define WEBRTC_AUDIO 1 # define WEBRTC_POSIX 1 # define WEBRTC_LEGACY 1 #elif defined(Q_OS_WIN) -# define WEBRTC_AUDIO 1 # define WEBRTC_WIN 1 # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 #elif defined(Q_OS_ANDROID) -// I don't yet have a working libwebrtc for android -// # define WEBRTC_AUDIO 1 // # define WEBRTC_POSIX 1 // # define WEBRTC_LEGACY 1 #elif defined(Q_OS_LINUX) && defined(Q_PROCESSOR_X86_64) -# define WEBRTC_AUDIO 1 # define WEBRTC_POSIX 1 #elif defined(Q_OS_LINUX) && defined(Q_PROCESSOR_ARM) -// WebRTC is basically impossible to build on aarch64 Linux. -// I am looking at https://gitlab.freedesktop.org/pulseaudio/webrtc-audio-processing for an alternative. -// # define WEBRTC_AUDIO 1 // # define WEBRTC_POSIX 1 // # define WEBRTC_LEGACY 1 #endif diff --git a/libraries/vircadia-client/CMakeLists.txt b/libraries/vircadia-client/CMakeLists.txt index 3443e4c90ef..399b332d26d 100644 --- a/libraries/vircadia-client/CMakeLists.txt +++ b/libraries/vircadia-client/CMakeLists.txt @@ -62,7 +62,7 @@ if(UNIX) target_link_libraries(${TARGET_NAME} Threads::Threads) endif() -link_hifi_libraries(shared networking) +link_hifi_libraries(shared networking audio-client-core pcm-codec opus-codec) add_subdirectory(tests) @@ -73,6 +73,7 @@ install(FILES src/context.h src/error.h src/messages.h + src/audio.h src/message_types.h src/node_list.h src/node_types.h diff --git a/libraries/vircadia-client/src/audio.cpp b/libraries/vircadia-client/src/audio.cpp new file mode 100644 index 00000000000..de59a3efa3d --- /dev/null +++ b/libraries/vircadia-client/src/audio.cpp @@ -0,0 +1,186 @@ +// +// audio.cpp +// libraries/vircadia-client/src +// +// Created by Nshan G. on 9 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "audio.h" + + +#include "internal/audio/AudioClient.h" +#include "internal/Error.h" +#include "internal/Context.h" + +using namespace vircadia::client; + + +VIRCADIA_CLIENT_DYN_API +int vircadia_enable_audio(int context_id) { + return chain(checkContextReady(context_id), [&](auto) { + std::next(std::begin(contexts), context_id)->audio().enable(); + return 0; + }); +} + +int checkAudioEnabled(int id) { + return chain(checkContextReady(id), [&](auto) { + return std::next(std::begin(contexts), id)->audio().isEnabled() + ? 0 + : toInt(ErrorCode::AUDIO_DISABLED); + }); +} + +int validateAudioFormat(const vircadia_audio_format& format, bool input = false) { + if ( + !(format.sample_type == AudioFormat::Signed16 || format.sample_type == AudioFormat::Float) || + format.sample_rate <= 0 || + format.channel_count <= 0 || + (input && format.channel_count > 2) + ) { + return toInt(ErrorCode::AUDIO_FORMAT_INVALID); + } else { + return 0; + } +} + +VIRCADIA_CLIENT_DYN_API +const char* vircadia_get_selected_audio_codec_name(int context_id) { + return chain(checkAudioEnabled(context_id), [&](auto) { + return std::next(std::begin(contexts), context_id)->audio().getSelectedCodecName().c_str(); + }); +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_codec_params(int context_id, const char* codec, vircadia_audio_codec_params params) { + return chain(checkAudioEnabled(context_id), [&](auto) { + auto& audio = std::next(std::begin(contexts), context_id)->audio(); + if (audio.setCodecAllowed(codec, params.allowed)) { + return 0; + } else { + return toInt(ErrorCode::AUDIO_CODEC_INVALID); + } + }); +} + +AudioFormat audioFormatFrom(const vircadia_audio_format& format) { + AudioFormat result{}; + result.sampleType = AudioFormat::SampleTag(format.sample_type); + result.sampleRate = format.sample_rate; + result.channelCount = format.channel_count; + return result; +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_format(int context_id, vircadia_audio_format format) { + return chain({ + checkAudioEnabled(context_id), + validateAudioFormat(format, true) + }, [&](auto) { + std::next(std::begin(contexts), context_id)->audio().setInput(audioFormatFrom(format)); + return 0; + }); +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_output_format(int context_id, vircadia_audio_format format) { + return chain({ + checkAudioEnabled(context_id), + validateAudioFormat(format) + }, [&](auto) { + std::next(std::begin(contexts), context_id)->audio().setOutput(audioFormatFrom(format)); + return 0; + }); +} + +VIRCADIA_CLIENT_DYN_API +uint8_t* vircadia_get_audio_input_context(int context_id) { + return chain(checkAudioEnabled(context_id), [&](auto) { + auto& audio = std::next(std::begin(contexts), context_id)->audio(); + return reinterpret_cast(audio.getInputContext()); + }); +} + +VIRCADIA_CLIENT_DYN_API +uint8_t* vircadia_get_audio_output_context(int context_id) { + return chain(checkAudioEnabled(context_id), [&](auto) { + auto& audio = std::next(std::begin(contexts), context_id)->audio(); + return reinterpret_cast(audio.getOutputContext()); + }); +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_data(uint8_t* audio_context, const uint8_t* data, int size) { + auto audioClient = reinterpret_cast(audio_context); + if (data == nullptr || size < 0) { + return toInt(ErrorCode::ARGUMENT_INVALID); + } else if (audioClient == nullptr) { + return toInt(ErrorCode::AUDIO_CONTEXT_INVALID); + } else { + audioClient->handleMicAudioInput(reinterpret_cast(data), size); + return 0; + } +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_get_audio_output_data(uint8_t* audio_context, uint8_t* data, int size) { + auto audioClient = reinterpret_cast(audio_context); + if (data == nullptr || size < 0) { + return toInt(ErrorCode::ARGUMENT_INVALID); + } else if (audioClient == nullptr) { + return toInt(ErrorCode::AUDIO_CONTEXT_INVALID); + } + + int bytesWritten = audioClient->getOutputIODevice().read(reinterpret_cast(data), size); + if (bytesWritten== -1) { + return toInt(ErrorCode::AUDIO_CONTEXT_INVALID); + } + + return bytesWritten; +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_bounds(int context_id, vircadia_bounds_ bounds) { + return chain(checkAudioEnabled(context_id), [&](auto) { + std::next(std::begin(contexts), context_id)->audio().setBounds(bounds); + return 0; + }); +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_vantage(int context_id, vircadia_vantage_ vantage) { + return chain(checkAudioEnabled(context_id), [&](auto) { + std::next(std::begin(contexts), context_id)->audio().setVantage(vantage); + return 0; + }); +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_echo(int context_id, uint8_t enabled) { + return chain(checkAudioEnabled(context_id), [&](auto) { + std::next(std::begin(contexts), context_id)->audio().setInputEcho(enabled); + return 0; + }); +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_muted(int context_id, uint8_t muted) { + return chain(checkAudioEnabled(context_id), [&](auto) { + std::next(std::begin(contexts), context_id)->audio().setIsMuted(muted); + return 0; + }); +} + +VIRCADIA_CLIENT_DYN_API +int vircadia_get_audio_input_muted_by_mixer(int context_id) { + return chain(checkAudioEnabled(context_id), [&](auto) { + return std::next(std::begin(contexts), context_id)->audio().getIsMutedByMixer() ? 1 : 0; + }); +} + + diff --git a/libraries/vircadia-client/src/audio.h b/libraries/vircadia-client/src/audio.h new file mode 100644 index 00000000000..9d37f0b1f93 --- /dev/null +++ b/libraries/vircadia-client/src/audio.h @@ -0,0 +1,324 @@ +// +// audio.h +// libraries/vircadia-client/src +// +// Created by Nshan G. on 9 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +/// @file + +#ifndef VIRCADIA_LIBRARIES_VIRCADIA_CLIENT_SRC_AUDIO_H +#define VIRCADIA_LIBRARIES_VIRCADIA_CLIENT_SRC_AUDIO_H + +#include + +#include "common.h" + +// FIXME: Avatar API also has these, so will need to move to common.h +// once merged. +struct vircadia_vector_ { + float x; + float y; + float z; +}; +struct vircadia_bounds_ { + vircadia_vector_ dimensions; + vircadia_vector_ offset; +}; +struct vircadia_quaternion_ { + float x; + float y; + float z; + float w; +}; +struct vircadia_vantage_ { + vircadia_vector_ position; + vircadia_quaternion_ rotation; +}; + +/// @brief Basic information of PCM audio data. +/// +/// Used to specify the format of audio input/output +/// (vircadia_set_audio_input_format()/vircadia_set_audio_output_format()). +/// For multiple channels the audio samples are interleaved in both cases. +struct vircadia_audio_format { + + /// @brief The type of a single sample. + /// + /// Valid values are defined in audio_constants.h + uint8_t sample_type; + + /// @brief The rate at which samples are processed. + /// + /// Specified as number of samples per second (Hz). This can be set to any + /// reasonable value (usually 8000-96000), typically 48000. The + /// input/output data will be re-sampled to/from desired network sample rate. + int sample_rate; + + /// @brief The number of channels. + /// + /// Input supports only mono(1 channel) and stereo(2 channels), and these + /// are sent to the mixer as is. \n + /// Output supports any value above zero (usually 1-8), but the incoming + /// mixer data is always stereo, so for other channel counts the data is + /// converted in a straight forward fashion (linear downmix, or upmix with + /// extra channels set to 0). + int channel_count; +}; + +/// @brief Audio codec parameters. +/// +/// Used with vircadia_set_audio_codec_params(). +struct vircadia_audio_codec_params { + + /// @brief Dis/allow the use of the codec for commination with the mixer. + /// + /// 0 - disallow \n + /// 1 - allow \n + /// + /// By default all codecs are allowed, and the mixer decides which to use + /// with the client (vircadia_get_selected_audio_codec_name()). This + /// parameter can be used to limit the choices for the mixer. + uint8_t allowed; + + // TODO: implement and document once + // https://github.com/vircadia/vircadia/pull/1582 is merged + uint8_t encoder_vbr; + uint8_t encoder_fec; + int encoder_bitrate; + int encoder_complexity; + int encoder_packet_loss; +}; + +/// @brief Enable handling of audio input and output. +/// +/// To start sending audio, the input the format must be specified with +/// vircadia_set_audio_input_format(), after which the data can be sent with +/// vircadia_set_audio_input_data(), with a input context retrieved from +/// vircadia_get_audio_input_context(). vircadia_set_audio_input_data() can be +/// called from a different thread with a valid input context. The input +/// context is invalidated if a new format is set (then a new context must be +/// retrieved before sending more data) or when entire client context is +/// destroyed. The audio output works in the same way: +/// vircadia_set_audio_output_format() -> vircadia_get_audio_output_context() +/// -> vircadia_get_audio_output_data(). +/// +/// @param context_id - The id of the client context (context.h). +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +VIRCADIA_CLIENT_DYN_API +int vircadia_enable_audio(int context_id); + +/// @brief Reads the audio codec selected for communication with the mixer. +/// +/// This is determined by the mixer, but the client can control allowed codecs +/// with vircadia_set_audio_codec_params(). \n +/// Possible values are defined in audio_constants.h. +/// +/// @param context_id - The id of the client context (context.h). +/// +/// @return A codec identifier, or an empty string if a codec has not been +/// selected yet, or null if an error occurred. +VIRCADIA_CLIENT_DYN_API +const char* vircadia_get_selected_audio_codec_name(int context_id); + +/// @brief Sets parameters of specified audio codec. +/// +/// @param context_id - The id of the client context (context.h). +/// @param codec - The id of the codec (audio_constants.h). +/// @param params - The codec parameters to set. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_codec_params(int context_id, const char* codec, vircadia_audio_codec_params params); + +/// @brief Sets the format of audio input data. +/// +/// Specifically it's the format of the data to be passed to +/// vircadia_set_audio_input_data(). \n +/// This function initiates a creation of new input context and invalidates any +/// previous input context. The new context can be retrieved using +/// vircadia_get_audio_input_context(). +/// +/// @param context_id - The id of the client context (context.h). +/// @param format - The format of audio input data to be sent. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() \n +/// vircadia_audio_format_invalid() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_format(int context_id, vircadia_audio_format format); + +/// @brief Sets the format of audio output data. +/// +/// Specifically it's the format of the data retrieved from +/// vircadia_get_audio_output_data(). \n +/// This function initiates a creation of new output context and invalidates +/// any previous output context. The new context can be retrieved using +/// vircadia_get_audio_output_context(). +/// +/// @param context_id - The id of the client context (context.h). +/// @param format - The format of audio output data to be received. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() \n +/// vircadia_audio_format_invalid() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_output_format(int context_id, vircadia_audio_format format); + +/// @brief Retrieves audio input context. +/// +/// This function must be polled to retrieve a new valid input context for +/// vircadia_set_audio_input_data(), after every call to +/// vircadia_set_audio_input_format(). +/// +/// @param context_id - The id of the client context (context.h). +/// +/// @return A pointer to the input context, or null if context is not yet +/// created/ready. +VIRCADIA_CLIENT_DYN_API +uint8_t* vircadia_get_audio_input_context(int context_id); + +/// @brief Retrieves audio output context. +/// +/// This function must be polled to retrieve a new valid output context for +/// vircadia_get_audio_output_data(), after every call to +/// vircadia_set_audio_output_format(). +/// +/// @param context_id - The id of the client context (context.h). +/// +/// @return A pointer to the output context, or null if context is not yet +/// created/ready. +VIRCADIA_CLIENT_DYN_API +uint8_t* vircadia_get_audio_output_context(int context_id); + +/// @brief Set the audio input data. +/// +/// Sets the audio input data to be sent to the mixer. The data is buffered +/// internally and sent out periodically on a different thread. This function +/// must be called with a valid audio input context +/// (vircadia_get_audio_input_context()), and data of specified audio format +/// (vircadia_set_audio_input_format()). This function can be called from a +/// different thread, to be used as a realtime audio callback. +/// +/// @param audio_context - The audio context. Must not be null. +/// @param data - The audio data to set. Must not be null. +/// @param size - The size of data to be set in bytes. Must not be negative. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_argument_invalid() \n +/// vircadia_error_audio_context_invalid() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_data(uint8_t* audio_context, const uint8_t* data, int size); + +/// @brief Get the audio output data. +/// +/// Retrieves the audio output data receives from the mixer. The data is +/// buffered internally as it arrives and must be periodically retrieved to not +/// exhaust the buffer. This function must be called with a valid audio output +/// context (vircadia_get_audio_output_context()), and data of specified audio +/// format (vircadia_set_audio_output_format()). This function can be called +/// from a different thread, to be used as a realtime audio callback. +/// +/// @param audio_context - The audio context. Must not be null. +/// @param data - The audio data buffer to read into. Must not be null. +/// @param size - The maximum size of data to read. Must not be negative. +/// +/// @return The number of bytes read, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_argument_invalid() \n +/// vircadia_error_audio_context_invalid() +VIRCADIA_CLIENT_DYN_API +int vircadia_get_audio_output_data(uint8_t* audio_context, uint8_t* data, int size); + +/// @brief Set the bounding box of the audio source. +/// +/// @param context_id - The id of the client context (context.h). +/// @param bounds - The bounds to set. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_bounds(int context_id, vircadia_bounds_ bounds); + +/// @brief Set the position and rotation of the audio source. +/// +/// @param context_id - The id of the client context (context.h). +/// @param vantage - The position and rotation (the vantage point) to set. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_vantage(int context_id, vircadia_vantage_ vantage); + +/// @brief Specify whether the audio input should be sent back to the client. +/// +/// By default the input is not sent back. +/// +/// @param context_id - The id of the client context (context.h). +/// @param enabled - 0 - not sent back, 1 - sent back. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_echo(int context_id, uint8_t enabled); + +/// @brief Un/mutes the audio input. +/// +/// By default the input not muted. +/// +/// @param context_id - The id of the client context (context.h). +/// @param muted - 0 - not muted, 1 - muted. +/// +/// @return non-negative value on success, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() +VIRCADIA_CLIENT_DYN_API +int vircadia_set_audio_input_muted(int context_id, uint8_t muted); + +/// @brief Determines whether this client has been muted by the mixer. +/// +/// @param context_id - The id of the client context (context.h). +/// +/// @return 1 - muted, 0 - not muted, or a negative error code. \n +/// Possible error codes: \n +/// vircadia_error_context_invalid() \n +/// vircadia_error_context_loss() \n +/// vircadia_error_audio_disabled() +VIRCADIA_CLIENT_DYN_API +int vircadia_get_audio_input_muted_by_mixer(int context_id); + +// TODO: API for injectors (AudioInjectorManager) +// TODO: API for soling, noise gate, noise reduction, gain, output buffer size (AudioPacketHandler) + +#endif /* end of include guard */ diff --git a/libraries/vircadia-client/src/audio_constants.cpp b/libraries/vircadia-client/src/audio_constants.cpp new file mode 100644 index 00000000000..7e51b3ca502 --- /dev/null +++ b/libraries/vircadia-client/src/audio_constants.cpp @@ -0,0 +1,39 @@ +// +// audio_constants.cpp +// libraries/vircadia-client/src +// +// Created by Nshan G. on 12 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "audio_constants.h" + +#include +#include +#include + + +VIRCADIA_CLIENT_DYN_API uint8_t vircadia_audio_sample_type_float() { + return AudioFormat::Float; +} + +VIRCADIA_CLIENT_DYN_API uint8_t vircadia_audio_sample_type_sint16() { + return AudioFormat::Signed16; +} + +VIRCADIA_CLIENT_DYN_API const char* vircadia_audio_codec_opus() { + return OpusCodec::getNameCString(); +} + +VIRCADIA_CLIENT_DYN_API const char* vircadia_audio_codec_pcm() { + return PCMCodec().getNameCString(); +} + +VIRCADIA_CLIENT_DYN_API const char* vircadia_audio_codec_zlib() { + return zLibCodec().getNameCString(); +} + diff --git a/libraries/vircadia-client/src/audio_constants.h b/libraries/vircadia-client/src/audio_constants.h new file mode 100644 index 00000000000..03427825b40 --- /dev/null +++ b/libraries/vircadia-client/src/audio_constants.h @@ -0,0 +1,49 @@ +// +// audio_constants.h +// libraries/vircadia-client/src +// +// Created by Nshan G. on 12 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +/// @file + +#ifndef LIBRARIES_VIRCADIA_CLIENT_SRC_MESSAGE_TYPES_H +#define LIBRARIES_VIRCADIA_CLIENT_SRC_MESSAGE_TYPES_H + +#include + +#include "common.h" + +/// @brief Value indicating floating point sample type of +/// vircadia_audio_format. +/// +/// @return Positive integer. +VIRCADIA_CLIENT_DYN_API uint8_t vircadia_audio_sample_type_float(); + +/// @brief Value indicating signed 16 bit sample type of +/// vircadia_audio_format. +/// +/// @return Positive integer. +VIRCADIA_CLIENT_DYN_API uint8_t vircadia_audio_sample_type_sint16(); + +/// @brief Unique identifier/name of the Opus codec. +/// +/// @return Null terminated string. +VIRCADIA_CLIENT_DYN_API const char* vircadia_audio_codec_opus(); + +/// @brief Unique identifier/name of the raw PCM codec. +/// +/// @return Null terminated string. +VIRCADIA_CLIENT_DYN_API const char* vircadia_audio_codec_pcm(); + +/// @brief Unique identifier/name of the zlib codec. +/// +/// @return Null terminated string. +VIRCADIA_CLIENT_DYN_API const char* vircadia_audio_codec_zlib(); + +#endif /* end of include guard */ diff --git a/libraries/vircadia-client/src/error.cpp b/libraries/vircadia-client/src/error.cpp index 070c7e40028..0d433f8c464 100644 --- a/libraries/vircadia-client/src/error.cpp +++ b/libraries/vircadia-client/src/error.cpp @@ -50,3 +50,19 @@ VIRCADIA_CLIENT_DYN_API int vircadia_error_packet_write() { VIRCADIA_CLIENT_DYN_API int vircadia_error_argument_invalid() { return toInt(ErrorCode::ARGUMENT_INVALID); } + +VIRCADIA_CLIENT_DYN_API int vircadia_error_audio_disabled() { + return toInt(ErrorCode::AUDIO_DISABLED); +} + +VIRCADIA_CLIENT_DYN_API int vircadia_audio_format_invalid() { + return toInt(ErrorCode::AUDIO_FORMAT_INVALID); +} + +VIRCADIA_CLIENT_DYN_API int vircadia_audio_context_invalid() { + return toInt(ErrorCode::AUDIO_CONTEXT_INVALID); +} + +VIRCADIA_CLIENT_DYN_API int vircadia_audio_codec_invalid() { + return toInt(ErrorCode::AUDIO_CODEC_INVALID); +} diff --git a/libraries/vircadia-client/src/error.h b/libraries/vircadia-client/src/error.h index 4a519813115..fd5b5812243 100644 --- a/libraries/vircadia-client/src/error.h +++ b/libraries/vircadia-client/src/error.h @@ -80,4 +80,39 @@ VIRCADIA_CLIENT_DYN_API int vircadia_error_packet_write(); /// @return -9 VIRCADIA_CLIENT_DYN_API int vircadia_error_argument_invalid(); +/// @brief Audio Functionality is disabled. +/// +/// Use vircadia_enable_audio() to enable audio. +/// +/// @return Unique negative error code. +VIRCADIA_CLIENT_DYN_API int vircadia_error_audio_disabled(); + +/// @brief Invalid audio format specified. +/// +/// Valid audio format must have non-zero positive frequency and +/// channel count, and one of sample types defined in +/// audio_constants.h. Additionally when passed to +/// vircadia_set_audio_input_format(), the channel count must not be +/// greater than two. +/// +/// @return Unique negative error code. +VIRCADIA_CLIENT_DYN_API int vircadia_audio_format_invalid(); + +/// @brief Invalid audio context specified. +/// +/// Audio context must be retrieved by polling +/// vircadia_get_audio_input_context()/vircadia_get_audio_output_context(), +/// after calling +/// vircadia_set_audio_input_format()/vircadia_set_audio_output_format(). +/// +/// @return Unique negative error code. +VIRCADIA_CLIENT_DYN_API int vircadia_audio_context_invalid(); + +/// @brief Invalid audio codec name specified. +/// +/// Valid codec names are defined in audio_constants.h. +/// +/// @return Unique negative error code. +VIRCADIA_CLIENT_DYN_API int vircadia_audio_codec_invalid(); + #endif /* end of include guard */ diff --git a/libraries/vircadia-client/src/internal/Audio.cpp b/libraries/vircadia-client/src/internal/Audio.cpp new file mode 100644 index 00000000000..e2051ecc800 --- /dev/null +++ b/libraries/vircadia-client/src/internal/Audio.cpp @@ -0,0 +1,129 @@ +// +// Audio.cpp +// libraries/vircadia-client/src/internal +// +// Created by Nshan G. on 4 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "Audio.h" + +#include +#include +#include + +#include +#include + +#include "audio/AudioClient.h" + +namespace vircadia::client +{ + + Audio::Audio() : + codecs{ + std::make_shared(), + std::make_shared(), + std::make_shared() + }, + allowedCodecs{codecs}, + selectedCodecName{""} + {} + + void Audio::enable() { + if (!isEnabled()) { + DependencyManager::set(allowedCodecs).data(); + } + } + + const std::vector>& Audio::getCodecs() { + return codecs; + } + + const std::string& Audio::getSelectedCodecName() { + selectedCodecName = DependencyManager::get()->getSelectedCodecName(); + return selectedCodecName; + } + + bool Audio::getIsMuted() const { + return DependencyManager::get()->getIsMuted(); + } + + bool Audio::getIsMutedByMixer() const { + return DependencyManager::get()->getIsMutedByMixer(); + } + + void Audio::setIsMuted(bool muted) { + DependencyManager::get()->setIsMuted(muted); + } + + void Audio::setVantage(const vircadia_vantage_& vantage) { + DependencyManager::get()->setVantage(vantage); + } + + void Audio::setBounds(const vircadia_bounds_& bounds) { + DependencyManager::get()->setBounds(bounds); + } + + void Audio::setInputEcho(bool enabled) { + DependencyManager::get()->setInputEcho(enabled); + } + + bool Audio::isEnabled() const { + return DependencyManager::isSet(); + } + + bool Audio::setCodecAllowed(std::string name, bool allow) { + auto matchName = [&name] (const auto& codecPtr) + { return codecPtr->getName().toStdString() == name; }; + + auto codec = std::find_if(codecs.begin(), codecs.end(), matchName); + if (codec == codecs.end()) { + return false; + } + + auto allowedCodec = std::find_if(allowedCodecs.begin(), allowedCodecs.end(), matchName); + auto isAllowed = allowedCodec != allowedCodecs.end(); + + if (allow != isAllowed) { + if (allow) { + allowedCodecs.push_back(*codec); + } else if (isAllowed) { + allowedCodecs.erase(allowedCodec); + } + + DependencyManager::get()->setSupportedCodecs(allowedCodecs); + } + + return true; + } + + void Audio::setInput(const AudioFormat& format) { + auto client = + DependencyManager::get(); + client->setInputFormat(format); + } + + void Audio::setOutput(const AudioFormat& format) { + DependencyManager::get()->setOutputFormat(format); + } + + AudioClient* Audio::getInputContext() { + return DependencyManager::get()->getInput(); + } + + AudioClient* Audio::getOutputContext() { + return DependencyManager::get()->getOutput(); + } + + void Audio::destroy() { + if (isEnabled()) { + DependencyManager::destroy(); + } + } + +} // namespace vircadia::client diff --git a/libraries/vircadia-client/src/internal/Audio.h b/libraries/vircadia-client/src/internal/Audio.h new file mode 100644 index 00000000000..6d3d8b97912 --- /dev/null +++ b/libraries/vircadia-client/src/internal/Audio.h @@ -0,0 +1,63 @@ +// +// Audio.h +// libraries/vircadia-client/src/internal +// +// Created by Nshan G. on 4 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef LIBRARIES_VIRCADIA_CLIENT_SRC_INTERNAL_AUDIO_H +#define LIBRARIES_VIRCADIA_CLIENT_SRC_INTERNAL_AUDIO_H + +#include +#include +#include + +class Codec; +struct AudioFormat; +struct vircadia_vantage_; +struct vircadia_bounds_; + +namespace vircadia::client { + + class AudioClient; + + /// @private + class Audio { + public: + Audio(); + void enable(); + bool isEnabled() const; + const std::vector>& getCodecs(); + bool setCodecAllowed(std::string name, bool allow); + + const std::string& getSelectedCodecName(); + bool getIsMuted() const; + bool getIsMutedByMixer() const; + void setIsMuted(bool); + void setVantage(const vircadia_vantage_&); + void setBounds(const vircadia_bounds_&); + void setInputEcho(bool enabled); + + void setInput(const AudioFormat&); + void setOutput(const AudioFormat&); + + AudioClient* getInputContext(); + AudioClient* getOutputContext(); + + void destroy(); + private: + + std::vector> codecs; + std::vector> allowedCodecs; + + std::string selectedCodecName; + }; + +} // namespace vircadia::client + +#endif /* end of include guard */ diff --git a/libraries/vircadia-client/src/internal/Context.cpp b/libraries/vircadia-client/src/internal/Context.cpp index 96c23af4fa9..0728522dca3 100644 --- a/libraries/vircadia-client/src/internal/Context.cpp +++ b/libraries/vircadia-client/src/internal/Context.cpp @@ -33,7 +33,8 @@ namespace vircadia::client { argc(1), argvData("qt_is_such_a_joke"), argv(&argvData[0]), - messages_() + messages_(), + audio_() { auto qtInitialization = qtInitialized.get_future(); appThread = std::thread{ [ this, nodeListParams, userAgent, info ] () { @@ -93,14 +94,23 @@ namespace vircadia::client { nodeList->getPacketReceiver().setShouldDropPackets(true); } + // TODO: a lot of objects destroyed below use + // deleteLater, and that doesn't seem to work + // reliably here (aboutToQuit signal), so maybe need + // to hand roll a quit event, to allow the event loop + // to clean things up, or not use deleteLater in this + // specific case + + audio_.destroy(); + QThreadPool::globalInstance()->clear(); QThreadPool::globalInstance()->waitForDone(); messages_.destroy(); - DependencyManager::destroy(); DependencyManager::destroy(); DependencyManager::destroy(); DependencyManager::destroy(); + DependencyManager::destroy(); }); @@ -166,6 +176,14 @@ namespace vircadia::client { return messages_; } + Audio& Context::audio() { + return audio_; + } + + const Audio& Context::audio() const { + return audio_; + } + std::list contexts; int checkContextValid(int id) { diff --git a/libraries/vircadia-client/src/internal/Context.h b/libraries/vircadia-client/src/internal/Context.h index 3fe0035cb37..21da813dda5 100644 --- a/libraries/vircadia-client/src/internal/Context.h +++ b/libraries/vircadia-client/src/internal/Context.h @@ -24,6 +24,7 @@ #include "../context.h" #include "Common.h" #include "Messages.h" +#include "Audio.h" class QCoreApplication; @@ -57,6 +58,9 @@ namespace vircadia::client { Messages& messages(); const Messages& messages() const; + Audio& audio(); + const Audio& audio() const; + private: std::thread appThread {}; std::atomic app {}; @@ -68,6 +72,7 @@ namespace vircadia::client { char* argv; Messages messages_; + Audio audio_; }; extern std::list contexts; diff --git a/libraries/vircadia-client/src/internal/Error.h b/libraries/vircadia-client/src/internal/Error.h index 94e380812d5..61c81fd4077 100644 --- a/libraries/vircadia-client/src/internal/Error.h +++ b/libraries/vircadia-client/src/internal/Error.h @@ -34,7 +34,12 @@ enum class ErrorCode : int { PACKET_WRITE, - ARGUMENT_INVALID + ARGUMENT_INVALID, + + AUDIO_DISABLED, + AUDIO_FORMAT_INVALID, + AUDIO_CONTEXT_INVALID, + AUDIO_CODEC_INVALID }; diff --git a/libraries/vircadia-client/src/internal/audio/AudioClient.cpp b/libraries/vircadia-client/src/internal/audio/AudioClient.cpp new file mode 100644 index 00000000000..eaec091a303 --- /dev/null +++ b/libraries/vircadia-client/src/internal/audio/AudioClient.cpp @@ -0,0 +1,189 @@ +// +// AudioClient.cpp +// libraries/vircadia-client/src/internal/audio +// +// Created by Nshan G. on 4 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "AudioClient.h" + +#include + +namespace vircadia::client +{ + // FIXME: avatar API PR also has these, need to be moved to a + // common header once merged + glm::vec3 glmVec3From_(vircadia_vector_ v) { + return {v.x, v.y, v.z}; + } + + glm::quat glmQuatFrom_(vircadia_quaternion_ q) { + return {q.w, q.x, q.y, q.z}; + } + + AudioClient::AudioClient(const std::vector>& supportedCodecs) : + AudioPacketHandler(), + position(), + orientation(), + codecs(supportedCodecs), + updateTimer(this), + codecsIn(codecs), + inputFormat{}, + outputFormat{}, + input(nullptr), + output(nullptr), + selectedCodecName(), + isMuted(false), + isMutedByMixer(false), + inputEcho(false), + vantage{{0.f, 0.f, 0.f}, {0.f,0.f,0.f,1.f}}, + bounds{{0.f, 0.f, 0.f}, {0.f, 0.f, 0.f}} + { + + setPositionGetter([this]() { return position; }); + setOrientationGetter([this]() { return orientation; }); + + setCustomDeleter([](Dependency* dependency){ + static_cast(dependency)->deleteLater(); + }); + + startThread(); + } + + void AudioClient::onStart() { + connect(&updateTimer, &QTimer::timeout, this, &AudioClient::update); + updateTimer.start(); + } + + void AudioClient::setInputFormat(AudioFormat format) { + std::scoped_lock lock(inout); + inputFormat = format; + input = nullptr; + } + + void AudioClient::setOutputFormat(AudioFormat format) { + std::scoped_lock lock(inout); + outputFormat = format; + output = nullptr; + } + + AudioClient* AudioClient::getInput() { + std::scoped_lock lock(inout); + return input; + } + + AudioClient* AudioClient::getOutput() { + std::scoped_lock lock(inout); + return output; + } + + void AudioClient::update() { + { + std::scoped_lock lock(inout); + + if (inputFormat != _inputFormat) { + cleanupInput(); + if (inputFormat.isValid()) { + setupInput(inputFormat); + } else if (!isDummyInput()) { + setupDummyInput(); + } + } + + if (_inputFormat.isValid()) { + input = this; + } + + if (outputFormat != _outputFormat) { + cleanupOutput(); + if (outputFormat.isValid()) { + setupOutput(outputFormat); + } + } + + if (_outputFormat.isValid()) { + output = this; + } + + selectedCodecName = _selectedCodecName.toStdString(); + + if (codecs != codecsIn) { + codecs = codecsIn; + negotiateAudioFormat(); + } + + position = glmVec3From_(vantage.position); + orientation = glmQuatFrom_(vantage.rotation); + avatarBoundingBoxCorner = glmVec3From_(bounds.offset); + avatarBoundingBoxScale = glmVec3From_(bounds.dimensions); + + _isMuted = isMuted || isMutedByMixer; + setServerEcho(inputEcho); + } + + if (_inputFormat.isValid()) { + sendInput(); + } + } + + void AudioClient::onMutedByMixer() { + std::scoped_lock lock(inout); + isMutedByMixer = true; + } + + void AudioClient::setSupportedCodecs(const std::vector>& supportedCodecs) { + std::scoped_lock lock(inout); + codecsIn = supportedCodecs; + } + + const std::vector>& AudioClient::getSupportedCodecs() const { + return codecs; + } + + AudioClient::AudioOutputIODevice& AudioClient::getOutputIODevice() { + return _audioOutputIODevice; + } + + std::string AudioClient::getSelectedCodecName() const { + std::scoped_lock lock(inout); + return _selectedCodecName.toStdString(); + } + + bool AudioClient::getIsMuted() const { + std::scoped_lock lock(inout); + return isMuted; + } + + bool AudioClient::getIsMutedByMixer() const { + std::scoped_lock lock(inout); + return isMutedByMixer; + } + + void AudioClient::setVantage(vircadia_vantage_ value) { + std::scoped_lock lock(inout); + vantage = value; + } + + void AudioClient::setBounds(vircadia_bounds_ value) { + std::scoped_lock lock(inout); + bounds = value; + } + + void AudioClient::setInputEcho(bool enabled) { + std::scoped_lock lock(inout); + inputEcho = enabled; + } + + void AudioClient::setIsMuted(bool muted) { + std::scoped_lock lock(inout); + isMuted = muted; + } + +} // namespace vircadia::client + +template class AudioPacketHandler; diff --git a/libraries/vircadia-client/src/internal/audio/AudioClient.h b/libraries/vircadia-client/src/internal/audio/AudioClient.h new file mode 100644 index 00000000000..075dcd1345b --- /dev/null +++ b/libraries/vircadia-client/src/internal/audio/AudioClient.h @@ -0,0 +1,87 @@ +// +// AudioClient.h +// libraries/vircadia-client/src/internal/audio +// +// Created by Nshan G. on 4 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef LIBRARIES_VIRCADIA_CLIENT_SRC_INTERNAL_AUDIO_AUDIOCLIENT_H +#define LIBRARIES_VIRCADIA_CLIENT_SRC_INTERNAL_AUDIO_AUDIOCLIENT_H + +#include + +#include + +#include "../../audio.h" + +namespace vircadia::client +{ + + /// @private + class AudioClient : public QObject, public Dependency, public AudioPacketHandler { + Q_OBJECT + public: + + AudioClient(const std::vector>& supportedCodecs); + void onStart(); + + void setSupportedCodecs(const std::vector>&); + const std::vector>& getSupportedCodecs() const; + + void setInputFormat(AudioFormat); + void setOutputFormat(AudioFormat); + + AudioClient* getInput(); + AudioClient* getOutput(); + + std::string getSelectedCodecName() const; + + bool getIsMuted() const; + bool getIsMutedByMixer() const; + + void setVantage(vircadia_vantage_); + void setBounds(vircadia_bounds_); + void setInputEcho(bool enabled); + void setIsMuted(bool muted); + + AudioOutputIODevice& getOutputIODevice(); + + private slots: + void update(); + + private: + + void onMutedByMixer(); + + friend class AudioPacketHandler; + + glm::vec3 position; + glm::quat orientation; + std::vector> codecs; + + QTimer updateTimer; + + std::vector> codecsIn; + AudioFormat inputFormat; + AudioFormat outputFormat; + AudioClient* input; + AudioClient* output; + std::string selectedCodecName; + bool isMuted; + bool isMutedByMixer; + bool inputEcho; + vircadia_vantage_ vantage; + vircadia_bounds_ bounds; + + + mutable std::mutex inout; + }; + +} // namespace vircadia::client + +#endif /* end of include guard */ diff --git a/libraries/vircadia-client/tests/CMakeLists.txt b/libraries/vircadia-client/tests/CMakeLists.txt index 880ec9649c7..de9547cbef4 100644 --- a/libraries/vircadia-client/tests/CMakeLists.txt +++ b/libraries/vircadia-client/tests/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5) find_package(Catch2 REQUIRED) -add_executable(vircadia-client-tests version.cpp client.cpp messages.cpp) +add_executable(vircadia-client-tests version.cpp client.cpp messages.cpp audio.cpp) target_link_libraries(vircadia-client-tests PRIVATE vircadia-client Catch2::Catch2WithMain) include(CTest) diff --git a/libraries/vircadia-client/tests/audio.cpp b/libraries/vircadia-client/tests/audio.cpp new file mode 100644 index 00000000000..678c6c9be37 --- /dev/null +++ b/libraries/vircadia-client/tests/audio.cpp @@ -0,0 +1,118 @@ +// +// audio.cpp +// libraries/vircadia-client/tests +// +// Created by Nshan G. on 9 July 2022. +// Copyright 2022 Vircadia contributors. +// Copyright 2022 DigiSomni LLC. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "../src/context.h" +#include "../src/node_list.h" +#include "../src/node_types.h" +#include "../src/audio.h" +#include "../src/audio_constants.h" +#include "../src/error.h" + +#include +#include +#include +#include + +#include + +#include "common.h" + +using namespace std::literals; + +TEST_CASE("Client API audio functionality.", "[client-api-audio]") { + + const float pi = std::acos(-1); + + const int context = vircadia_create_context(vircadia_context_defaults()); + vircadia_connect(context, "localhost"); + + { + std::thread input_thread{}; + std::thread output_thread{}; + + auto cleanup = defer([context, &input_thread, &output_thread](){ + REQUIRE(vircadia_destroy_context(context) == 0); + if (input_thread.joinable()) { + input_thread.join(); + } + if (output_thread.joinable()) { + output_thread.join(); + } + }); + + + bool input_started = false; + uint8_t* input = nullptr; + bool output_started = false; + uint8_t* output = nullptr; + std::vector output_data; + + REQUIRE(vircadia_enable_audio(context) >= 0); + REQUIRE(vircadia_set_audio_input_echo(context, 1) >= 0); + REQUIRE(vircadia_set_audio_input_format(context, vircadia_audio_format{ + vircadia_audio_sample_type_float(), 48000, 1}) >= 0); + REQUIRE(vircadia_set_audio_output_format(context, vircadia_audio_format{ + vircadia_audio_sample_type_float(), 48000, 2}) >= 0); + + for (int i = 0; i < 300; ++i) { + int status = vircadia_connection_status(context); + if (status == 1) { + + if (input == nullptr) { + input = vircadia_get_audio_input_context(context); + } else if (!input_started) { + input_thread = std::thread([input, pi, current_sample = 0]() mutable { + constexpr int sample_size = 480; + static float data[sample_size]; + while (current_sample != 48000) { + for (int i = 0; i < sample_size; ++i) { + data[i] = std::sin(148 * 2 * pi * ((current_sample%48000)/48000.f)); + ++current_sample; + } + REQUIRE(vircadia_set_audio_input_data(input, + reinterpret_cast(data), sample_size * sizeof(float)) >= 0); + std::this_thread::sleep_for(10ms); + } + }); + input_started = true; + } + + if (output == nullptr) { + output = vircadia_get_audio_output_context(context); + } else if(!output_started) { + output_thread = std::thread([output, &output_data]() { + constexpr int sample_size = 480 * 2; + while (output_data.size() != 48000 * 2) { + output_data.resize(output_data.size() + sample_size); + REQUIRE(vircadia_get_audio_output_data(output, + reinterpret_cast(output_data.data() + output_data.size() - sample_size), + sample_size * sizeof(float)) == sample_size * sizeof(float)); + std::this_thread::sleep_for(10ms); + } + }); + output_started = true; + } + + } + + std::this_thread::sleep_for(10ms); + } + + if (!output_data.empty()) { + // TODO: check this as data arrives and finish the test once satisfied + REQUIRE(std::count_if(output_data.begin(), output_data.end(), + [](auto x) { return std::abs(x) > 0.5f; }) > 5000); + } + + } + +} diff --git a/plugins/opusCodec/CMakeLists.txt b/plugins/opusCodec/CMakeLists.txt index 583aff85d66..617f4c57433 100644 --- a/plugins/opusCodec/CMakeLists.txt +++ b/plugins/opusCodec/CMakeLists.txt @@ -8,7 +8,7 @@ set(TARGET_NAME opusCodec) setup_hifi_client_server_plugin() -link_hifi_libraries(shared audio plugins) +link_hifi_libraries(shared audio plugins opus-codec) target_opus() if (BUILD_SERVER) diff --git a/plugins/opusCodec/src/OpusCodecManager.cpp b/plugins/opusCodec/src/OpusCodecManager.cpp index 1e3d73a2295..d2123ba7589 100644 --- a/plugins/opusCodec/src/OpusCodecManager.cpp +++ b/plugins/opusCodec/src/OpusCodecManager.cpp @@ -1,5 +1,5 @@ // -// opusCodec.cpp +// OpusCodecManager.cpp // plugins/opusCodec/src // // Created by Michael Bailey on 12/20/2019 @@ -14,11 +14,8 @@ #include #include - -#include "OpusEncoder.h" -#include "OpusDecoder.h" - -const char* AthenaOpusCodec::NAME { "opus" }; +#include +#include void AthenaOpusCodec::init() { } @@ -40,19 +37,23 @@ bool AthenaOpusCodec::isSupported() const { return true; } +const QString AthenaOpusCodec::getName() const { + return OpusCodec::getName(); +} + Encoder* AthenaOpusCodec::createEncoder(int sampleRate, int numChannels) { - return new AthenaOpusEncoder(sampleRate, numChannels); + return OpusCodec::createEncoder(sampleRate, numChannels); } Decoder* AthenaOpusCodec::createDecoder(int sampleRate, int numChannels) { - return new AthenaOpusDecoder(sampleRate, numChannels); + return OpusCodec::createDecoder(sampleRate, numChannels); } void AthenaOpusCodec::releaseEncoder(Encoder* encoder) { - delete encoder; + return OpusCodec::releaseEncoder(encoder); } void AthenaOpusCodec::releaseDecoder(Decoder* decoder) { - delete decoder; + return OpusCodec::releaseDecoder(decoder); } diff --git a/plugins/opusCodec/src/OpusCodecManager.h b/plugins/opusCodec/src/OpusCodecManager.h index be6a6b2ff0e..597bd5bf2bb 100644 --- a/plugins/opusCodec/src/OpusCodecManager.h +++ b/plugins/opusCodec/src/OpusCodecManager.h @@ -13,14 +13,15 @@ #define hifi__OpusCodecManager_h #include +#include -class AthenaOpusCodec : public CodecPlugin { +class AthenaOpusCodec : public CodecPlugin, public OpusCodec { Q_OBJECT public: // Plugin functions bool isSupported() const override; - const QString getName() const override { return NAME; } + const QString getName() const override; void init() override; void deinit() override; @@ -34,9 +35,6 @@ class AthenaOpusCodec : public CodecPlugin { virtual Decoder* createDecoder(int sampleRate, int numChannels) override; virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; - -private: - static const char* NAME; }; #endif // hifi__opusCodecManager_h diff --git a/plugins/pcmCodec/CMakeLists.txt b/plugins/pcmCodec/CMakeLists.txt index 34e49d908ba..029fb80e4ed 100644 --- a/plugins/pcmCodec/CMakeLists.txt +++ b/plugins/pcmCodec/CMakeLists.txt @@ -8,7 +8,7 @@ set(TARGET_NAME pcmCodec) setup_hifi_client_server_plugin() -link_hifi_libraries(shared audio plugins) +link_hifi_libraries(shared audio plugins pcm-codec) if (BUILD_SERVER) install_beside_console() diff --git a/plugins/pcmCodec/src/PCMCodecManager.cpp b/plugins/pcmCodec/src/PCMCodecManager.cpp index 04adb367afe..59ca714bc4c 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.cpp +++ b/plugins/pcmCodec/src/PCMCodecManager.cpp @@ -15,80 +15,84 @@ #include -const char* PCMCodec::NAME { "pcm" }; - -void PCMCodec::init() { +void PCMCodecManager::init() { } -void PCMCodec::deinit() { +void PCMCodecManager::deinit() { } -bool PCMCodec::activate() { +bool PCMCodecManager::activate() { CodecPlugin::activate(); return true; } -void PCMCodec::deactivate() { +void PCMCodecManager::deactivate() { CodecPlugin::deactivate(); } -bool PCMCodec::isSupported() const { +bool PCMCodecManager::isSupported() const { return true; } +const QString PCMCodecManager::getName() const { + return PCMCodec::getName(); +} -Encoder* PCMCodec::createEncoder(int sampleRate, int numChannels) { - return this; -} -Decoder* PCMCodec::createDecoder(int sampleRate, int numChannels) { - return this; +Encoder* PCMCodecManager::createEncoder(int sampleRate, int numChannels) { + return PCMCodec::createEncoder(sampleRate, numChannels); } -void PCMCodec::releaseEncoder(Encoder* encoder) { - // do nothing +Decoder* PCMCodecManager::createDecoder(int sampleRate, int numChannels) { + return PCMCodec::createDecoder(sampleRate, numChannels); } -void PCMCodec::releaseDecoder(Decoder* decoder) { - // do nothing +void PCMCodecManager::releaseEncoder(Encoder* encoder) { + PCMCodec::releaseEncoder(encoder); } -const char* zLibCodec::NAME { "zlib" }; +void PCMCodecManager::releaseDecoder(Decoder* decoder) { + PCMCodec::releaseDecoder(decoder); +} -void zLibCodec::init() { +void zLibCodecManager::init() { } -void zLibCodec::deinit() { +void zLibCodecManager::deinit() { } -bool zLibCodec::activate() { +bool zLibCodecManager::activate() { CodecPlugin::activate(); return true; } -void zLibCodec::deactivate() { +void zLibCodecManager::deactivate() { CodecPlugin::deactivate(); } -bool zLibCodec::isSupported() const { +bool zLibCodecManager::isSupported() const { return true; } -Encoder* zLibCodec::createEncoder(int sampleRate, int numChannels) { - return this; +const QString zLibCodecManager::getName() const { + return zLibCodec::getName(); +} + +Encoder* zLibCodecManager::createEncoder(int sampleRate, int numChannels) { + return zLibCodec::createEncoder(sampleRate, numChannels); } -Decoder* zLibCodec::createDecoder(int sampleRate, int numChannels) { - return this; +Decoder* zLibCodecManager::createDecoder(int sampleRate, int numChannels) { + return zLibCodec::createDecoder(sampleRate, numChannels); } -void zLibCodec::releaseEncoder(Encoder* encoder) { - // do nothing... it wasn't allocated +void zLibCodecManager::releaseEncoder(Encoder* encoder) { + return zLibCodec::releaseEncoder(encoder); } -void zLibCodec::releaseDecoder(Decoder* decoder) { - // do nothing... it wasn't allocated +void zLibCodecManager::releaseDecoder(Decoder* decoder) { + return zLibCodec::releaseDecoder(decoder); } diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index 178f7cbd9b7..b673eb2c587 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -13,15 +13,15 @@ #define hifi__PCMCodecManager_h #include -#include +#include -class PCMCodec : public CodecPlugin, public Encoder, public Decoder { +class PCMCodecManager : public CodecPlugin, public PCMCodec { Q_OBJECT public: // Plugin functions bool isSupported() const override; - const QString getName() const override { return NAME; } + const QString getName() const override; void init() override; void deinit() override; @@ -35,31 +35,15 @@ class PCMCodec : public CodecPlugin, public Encoder, public Decoder { virtual Decoder* createDecoder(int sampleRate, int numChannels) override; virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; - - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = decodedBuffer; - } - - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = encodedBuffer; - } - - virtual void lostFrame(QByteArray& decodedBuffer) override { - decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); - memset(decodedBuffer.data(), 0, decodedBuffer.size()); - } - -private: - static const char* NAME; }; -class zLibCodec : public CodecPlugin, public Encoder, public Decoder { +class zLibCodecManager : public CodecPlugin, public zLibCodec { Q_OBJECT public: // Plugin functions bool isSupported() const override; - const QString getName() const override { return NAME; } + const QString getName() const override; void init() override; void deinit() override; @@ -73,22 +57,6 @@ class zLibCodec : public CodecPlugin, public Encoder, public Decoder { virtual Decoder* createDecoder(int sampleRate, int numChannels) override; virtual void releaseEncoder(Encoder* encoder) override; virtual void releaseDecoder(Decoder* decoder) override; - - virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = qCompress(decodedBuffer); - } - - virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { - decodedBuffer = qUncompress(encodedBuffer); - } - - virtual void lostFrame(QByteArray& decodedBuffer) override { - decodedBuffer.resize(AudioConstants::NETWORK_FRAME_BYTES_STEREO); - memset(decodedBuffer.data(), 0, decodedBuffer.size()); - } - -private: - static const char* NAME; }; #endif // hifi__PCMCodecManager_h diff --git a/plugins/pcmCodec/src/PCMCodecProvider.cpp b/plugins/pcmCodec/src/PCMCodecProvider.cpp index dded40bfc98..9d65c0f2ca7 100644 --- a/plugins/pcmCodec/src/PCMCodecProvider.cpp +++ b/plugins/pcmCodec/src/PCMCodecProvider.cpp @@ -30,12 +30,12 @@ class PCMCodecProvider : public QObject, public CodecProvider { static std::once_flag once; std::call_once(once, [&] { - CodecPluginPointer pcmCodec(std::make_shared()); + CodecPluginPointer pcmCodec(std::make_shared()); if (pcmCodec->isSupported()) { _codecPlugins.push_back(pcmCodec); } - CodecPluginPointer zlibCodec(std::make_shared()); + CodecPluginPointer zlibCodec(std::make_shared()); if (zlibCodec->isSupported()) { _codecPlugins.push_back(zlibCodec); }