Native client library audio mixer APIs.

CMakeLists.txt

@@ -227,11 +227,19 @@ set(BUILD_INSTALLER_OPTION ON)
 set(DISABLE_QML_OPTION OFF)
 set(DOWNLOAD_SERVERLESS_CONTENT_OPTION OFF)
 set(ENABLE_WEBRTC_DATA_CHANNELS_OPTION OFF)
+set(ENABLE_WEBRTC_AUDIO_OPTION OFF)
 
 if (WIN32 OR (UNIX AND NOT APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64"))
     set(ENABLE_WEBRTC_DATA_CHANNELS_OPTION ON)
 endif()
 
+if (WIN32 OR (UNIX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64"))
+    # We don't yet have a working libwebrtc for android.
+    # WebRTC is difficult to build on aarch64 Linux.
+    # an alternative is https://gitlab.freedesktop.org/pulseaudio/webrtc-audio-processing.
+    set(ENABLE_WEBRTC_AUDIO_OPTION ON)
+endif()
+
 if (ANDROID OR UWP)
   set(BUILD_SERVER_OPTION OFF)
   set(BUILD_TOOLS_OPTION OFF)
@@ -304,13 +312,18 @@ option(
   ${DOWNLOAD_SERVERLESS_CONTENT_OPTION}
 )
 option(ENABLE_WEBRTC_DATA_CHANNELS "Use WebRTC for client-server connection in parallel with UDP" ${ENABLE_WEBRTC_DATA_CHANNELS_OPTION})
+option(ENABLE_WEBRTC_AUDIO "Use WebRTC for audio processing (e.g. echo cancelation)." ${ENABLE_WEBRTC_AUDIO_OPTION})
 
 set(PLATFORM_QT_GL OpenGL)
 
 if (ENABLE_WEBRTC_DATA_CHANNELS)
   add_compile_definitions(WEBRTC_DATA_CHANNELS=1)
 endif()
 
+if (ENABLE_WEBRTC_AUDIO)
+  add_compile_definitions(WEBRTC_AUDIO=1)
+endif()
+
 if (USE_KHR_ROBUSTNESS)
   add_definitions(-DUSE_KHR_ROBUSTNESS)
 endif()
@@ -338,6 +351,7 @@ MESSAGE(STATUS "Build installer:       " ${BUILD_INSTALLER})
 MESSAGE(STATUS "GL ES:                 " ${USE_GLES})
 MESSAGE(STATUS "DL serverless content: " ${DOWNLOAD_SERVERLESS_CONTENT})
 MESSAGE(STATUS "WebRTC Data Channels:  " ${ENABLE_WEBRTC_DATA_CHANNELS})
+MESSAGE(STATUS "WebRTC Audio:          " ${ENABLE_WEBRTC_AUDIO})
 MESSAGE(STATUS "Static standard libraries: " ${STATIC_STDLIB})
 MESSAGE(STATUS "Shared internal libraries: " ${BUILD_SHARED_LIBS})
 

assignment-client/src/Agent.cpp

@@ -475,11 +475,6 @@ void Agent::executeScript() {
                 packetType = PacketType::SilentAudioFrame;
             }
 
-            Transform audioTransform;
-            auto headOrientation = scriptedAvatar->getHeadOrientation();
-            audioTransform.setTranslation(scriptedAvatar->getWorldPosition());
-            audioTransform.setRotation(headOrientation);
-
             QByteArray encodedBuffer;
             if (_encoder) {
                 _encoder->encode(audio, encodedBuffer);
@@ -488,7 +483,7 @@ void Agent::executeScript() {
             }
 
             AbstractAudioInterface::emitAudioPacket(encodedBuffer.data(), encodedBuffer.size(), audioSequenceNumber, false,
-                                                    audioTransform, scriptedAvatar->getWorldPosition(), glm::vec3(0),
+                                                    {scriptedAvatar->getWorldPosition(), scriptedAvatar->getHeadOrientation()}, scriptedAvatar->getWorldPosition(), glm::vec3(0),
                                                     packetType, _selectedCodecName);
         });
 
@@ -719,7 +714,7 @@ void Agent::processAgentAvatarAudio() {
             if (isPlayingRecording && !_shouldMuteRecordingAudio) {
                 _shouldMuteRecordingAudio = true;
             }
-            
+
             auto audioData = _avatarSound->getAudioData();
             nextSoundOutput = reinterpret_cast<const int16_t*>(audioData->rawData()
                     + _numAvatarSoundSentBytes);
@@ -880,7 +875,7 @@ void Agent::aboutToFinish() {
     {
         DependencyManager::get<ScriptEngines>()->shutdownScripting();
     }
-    
+
     DependencyManager::destroy<ScriptEngines>();
 
     DependencyManager::destroy<AssignmentDynamicFactory>();

assignment-client/src/audio/AudioMixerSlave.cpp

@@ -279,7 +279,7 @@ bool shouldBeSkipped(MixableStream& stream, const Node& listener,
     bool shouldCheckIgnoreBox = (listenerAudioStream.isIgnoreBoxEnabled() ||
                                  stream.positionalStream->isIgnoreBoxEnabled());
     if (shouldCheckIgnoreBox &&
-        listenerAudioStream.getIgnoreBox().touches(stream.positionalStream->getIgnoreBox())) {
+        isTouching(listenerAudioStream.getIgnoreBox(), stream.positionalStream->getIgnoreBox())) {
         return true;
     }
 
@@ -587,7 +587,7 @@ void AudioMixerSlave::updateHRTFParameters(AudioMixerClientData::MixableStream&
     glm::vec3 relativePosition = streamToAdd->getPosition() - listeningNodeStream.getPosition();
 
     float distance = glm::max(glm::length(relativePosition), EPSILON);
-    float gain = isEcho ? 1.0f : computeGain(masterAvatarGain, masterInjectorGain, listeningNodeStream, *streamToAdd, 
+    float gain = isEcho ? 1.0f : computeGain(masterAvatarGain, masterInjectorGain, listeningNodeStream, *streamToAdd,
                                              relativePosition, distance);
     float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition);
 
@@ -815,14 +815,14 @@ float computeAzimuth(const AvatarAudioStream& listeningNodeStream,
 
     float rotatedSourcePositionLength2 = glm::length2(rotatedSourcePosition);
     if (rotatedSourcePositionLength2 > SOURCE_DISTANCE_THRESHOLD) {
-        
+
         // produce an oriented angle about the y-axis
         glm::vec3 direction = rotatedSourcePosition * (1.0f / fastSqrtf(rotatedSourcePositionLength2));
         float angle = fastAcosf(glm::clamp(-direction.z, -1.0f, 1.0f)); // UNIT_NEG_Z is "forward"
         return (direction.x < 0.0f) ? -angle : angle;
 
-    } else {   
+    } else {
         // no azimuth if they are in same spot
-        return 0.0f; 
+        return 0.0f;
     }
 }

interface/src/Application.cpp

@@ -7387,6 +7387,7 @@ void Application::nodeActivated(SharedNodePointer node) {
         }
     }
 
+    // FIXME: this will be handled in AudioClient, once it's derived from AudioPacketHandler
     if (node->getType() == NodeType::AudioMixer && !isInterstitialMode()) {
         DependencyManager::get<AudioClient>()->negotiateAudioFormat();
     }

libraries/audio-client-core/CMakeLists.txt

@@ -0,0 +1,22 @@
+set(TARGET_NAME audio-client-core)
+if (ANDROID)
+  set(PLATFORM_QT_COMPONENTS AndroidExtras)
+endif ()
+setup_hifi_library(Network)
+link_hifi_libraries(audio)
+include_hifi_library_headers(shared)
+include_hifi_library_headers(networking)
+
+if (ENABLE_WEBRTC_AUDIO)
+  target_webrtc()
+endif ()
+
+# append audio includes to our list of includes to bubble
+target_include_directories(${TARGET_NAME} PUBLIC "${HIFI_LIBRARY_DIR}/audio/src")
+
+# have CMake grab externals for us
+if (APPLE)
+  find_library(CoreAudio CoreAudio)
+  find_library(CoreFoundation CoreFoundation)
+  target_link_libraries(${TARGET_NAME} ${CoreAudio} ${CoreFoundation})
+endif ()

libraries/audio-client-core/src/AudioClientLogging.cpp

@@ -0,0 +1,14 @@
+//
+//  AudioClientLogging.cpp
+//  libraries/audio-client/src
+//
+//  Created by Seth Alves on 4/6/15.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "AudioClientLogging.h"
+
+Q_LOGGING_CATEGORY(audioclient, "hifi.audioclient")

libraries/audio-client-core/src/AudioClientLogging.h

@@ -0,0 +1,20 @@
+//
+//  AudioClientLogging.h
+//  libraries/audio-client/src
+//
+//  Created by Seth Alves on 4/6/15.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AudioClientLogging_h
+#define hifi_AudioClientLogging_h
+
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(audioclient)
+
+#endif // hifi_AudioClientLogging_h
+

libraries/audio-client-core/src/AudioIOStats.cpp

@@ -0,0 +1,209 @@
+//
+//  AudioStats.cpp
+//  interface/src/audio
+//
+//  Created by Stephen Birarda on 2014-12-16.
+//  Copyright 2014 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "AudioIOStats.h"
+
+#include <AudioConstants.h>
+#include <MixedProcessedAudioStream.h>
+#include <NodeList.h>
+#include <PositionalAudioStream.h>
+
+// This is called 1x/sec (see AudioClient) and we want it to log the last 5s
+static const int INPUT_READS_WINDOW = 5;
+static const int INPUT_UNPLAYED_WINDOW = 5;
+static const int OUTPUT_UNPLAYED_WINDOW = 5;
+
+static const int APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS = (int)(30.0f * 1000.0f / AudioConstants::NETWORK_FRAME_MSECS);
+
+
+AudioIOStats::AudioIOStats(MixedProcessedAudioStream* receivedAudioStream) :
+    _interface(new AudioStatsInterface(this)),
+    _inputMsRead(1, INPUT_READS_WINDOW),
+    _inputMsUnplayed(1, INPUT_UNPLAYED_WINDOW),
+    _outputMsUnplayed(1, OUTPUT_UNPLAYED_WINDOW),
+    _lastSentPacketTime(0),
+    _packetTimegaps(1, APPROXIMATELY_30_SECONDS_OF_AUDIO_PACKETS),
+    _receivedAudioStream(receivedAudioStream)
+{
+
+}
+
+void AudioIOStats::reset() {
+    _receivedAudioStream->resetStats();
+
+    _inputMsRead.reset();
+    _inputMsUnplayed.reset();
+    _outputMsUnplayed.reset();
+    _packetTimegaps.reset();
+
+    _interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps);
+    _interface->updateMixerStream(AudioStreamStats());
+    _interface->updateClientStream(AudioStreamStats());
+    _interface->updateInjectorStreams(QHash<QUuid, AudioStreamStats>());
+}
+
+void AudioIOStats::sentPacket() const {
+    // first time this is 0
+    if (_lastSentPacketTime == 0) {
+        _lastSentPacketTime = usecTimestampNow();
+    } else {
+        quint64 now = usecTimestampNow();
+        quint64 gap = now - _lastSentPacketTime;
+        _lastSentPacketTime = now;
+        _packetTimegaps.update(gap);
+    }
+}
+
+void AudioIOStats::processStreamStatsPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer sendingNode) {
+    // parse the appendFlag, clear injected audio stream stats if 0
+    quint8 appendFlag;
+    message->readPrimitive(&appendFlag);
+
+    if (appendFlag & AudioStreamStats::START) {
+        _injectorStreams.clear();
+    }
+
+    // parse the number of stream stats structs to follow
+    quint16 numStreamStats;
+    message->readPrimitive(&numStreamStats);
+
+    // parse the stream stats
+    AudioStreamStats streamStats;
+    for (quint16 i = 0; i < numStreamStats; i++) {
+        message->readPrimitive(&streamStats);
+
+        if (streamStats._streamType == PositionalAudioStream::Microphone) {
+            _interface->updateMixerStream(streamStats);
+        } else {
+            _injectorStreams[streamStats._streamIdentifier] = streamStats;
+        }
+    }
+
+    if (appendFlag & AudioStreamStats::END) {
+        _interface->updateInjectorStreams(_injectorStreams);
+    }
+}
+
+void AudioIOStats::publish() {
+    // call _receivedAudioStream's per-second callback
+    _receivedAudioStream->perSecondCallbackForUpdatingStats();
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer);
+    if (!audioMixer) {
+        return;
+    }
+
+    quint8 appendFlag = AudioStreamStats::START | AudioStreamStats::END;
+    quint16 numStreamStatsToPack = 1;
+    AudioStreamStats stats = _receivedAudioStream->getAudioStreamStats();
+
+    // update the interface
+    _interface->updateLocalBuffers(_inputMsRead, _inputMsUnplayed, _outputMsUnplayed, _packetTimegaps);
+    _interface->updateClientStream(stats);
+
+    // prepare a packet to the mixer
+    int statsPacketSize = sizeof(appendFlag) + sizeof(numStreamStatsToPack) + sizeof(stats);
+    auto statsPacket = NLPacket::create(PacketType::AudioStreamStats, statsPacketSize);
+
+    // pack append flag
+    statsPacket->writePrimitive(appendFlag);
+
+    // pack number of stats packed
+    statsPacket->writePrimitive(numStreamStatsToPack);
+
+    // pack downstream audio stream stats
+    statsPacket->writePrimitive(stats);
+
+    // send packet
+    nodeList->sendPacket(std::move(statsPacket), *audioMixer);
+}
+
+AudioStreamStatsInterface::AudioStreamStatsInterface(QObject* parent) :
+    QObject(parent) {}
+
+void AudioStreamStatsInterface::updateStream(const AudioStreamStats& stats) {
+    lossRate(stats._packetStreamStats.getLostRate());
+    lossCount(stats._packetStreamStats._lost);
+    lossRateWindow(stats._packetStreamWindowStats.getLostRate());
+    lossCountWindow(stats._packetStreamWindowStats._lost);
+
+    framesDesired(stats._desiredJitterBufferFrames);
+    framesAvailable(stats._framesAvailable);
+    framesAvailableAvg(stats._framesAvailableAverage);
+
+    unplayedMsMax(stats._unplayedMs);
+
+    starveCount(stats._starveCount);
+    lastStarveDurationCount(stats._consecutiveNotMixedCount);
+    dropCount(stats._framesDropped);
+    overflowCount(stats._overflowCount);
+
+    timegapMsMax(stats._timeGapMax / USECS_PER_MSEC);
+    timegapMsAvg(stats._timeGapAverage / USECS_PER_MSEC);
+    timegapMsMaxWindow(stats._timeGapWindowMax / USECS_PER_MSEC);
+    timegapMsAvgWindow(stats._timeGapWindowAverage / USECS_PER_MSEC);
+}
+
+AudioStatsInterface::AudioStatsInterface(QObject* parent) :
+    QObject(parent),
+    _client(new AudioStreamStatsInterface(this)),
+    _mixer(new AudioStreamStatsInterface(this)),
+    _injectors(new QObject(this)) {}
+
+
+void AudioStatsInterface::updateLocalBuffers(const MovingMinMaxAvg<float>& inputMsRead,
+    const MovingMinMaxAvg<float>& inputMsUnplayed,
+    const MovingMinMaxAvg<float>& outputMsUnplayed,
+    const MovingMinMaxAvg<quint64>& timegaps) {
+    if (SharedNodePointer audioNode = DependencyManager::get<NodeList>()->soloNodeOfType(NodeType::AudioMixer)) {
+        pingMs(audioNode->getPingMs());
+    }
+
+    inputReadMsMax(inputMsRead.getWindowMax());
+    inputUnplayedMsMax(inputMsUnplayed.getWindowMax());
+    outputUnplayedMsMax(outputMsUnplayed.getWindowMax());
+
+    sentTimegapMsMax(timegaps.getMax() / USECS_PER_MSEC);
+    sentTimegapMsAvg(timegaps.getAverage() / USECS_PER_MSEC);
+    sentTimegapMsMaxWindow(timegaps.getWindowMax() / USECS_PER_MSEC);
+    sentTimegapMsAvgWindow(timegaps.getWindowAverage() / USECS_PER_MSEC);
+}
+
+void AudioStatsInterface::updateInjectorStreams(const QHash<QUuid, AudioStreamStats>& stats) {
+    // Get existing injectors
+    auto injectorIds = _injectors->dynamicPropertyNames();
+
+    // Go over reported injectors
+    QHash<QUuid, AudioStreamStats>::const_iterator injector = stats.constBegin();
+    while (injector != stats.constEnd()) {
+        const auto id = injector.key().toByteArray();
+        // Mark existing injector (those left will be removed)
+        injectorIds.removeOne(id);
+        auto injectorProperty = _injectors->property(id);
+        // Add new injector
+        if (!injectorProperty.isValid()) {
+            injectorProperty = QVariant::fromValue(new AudioStreamStatsInterface(this));
+            _injectors->setProperty(id, injectorProperty);
+        }
+        // Update property with reported injector
+        injectorProperty.value<AudioStreamStatsInterface*>()->updateStream(injector.value());
+        ++injector;
+    }
+
+    // Remove unreported injectors
+    for (auto& id : injectorIds) {
+        _injectors->property(id).value<AudioStreamStatsInterface*>()->deleteLater();
+        _injectors->setProperty(id, QVariant());
+    }
+
+    emit injectorStreamsChanged();
+}