diff --git a/assignment-client/src/audio/AudioMixerSlavePool.cpp b/assignment-client/src/audio/AudioMixerSlavePool.cpp index e8a2909acbf..966ab2897d7 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.cpp +++ b/assignment-client/src/audio/AudioMixerSlavePool.cpp @@ -18,65 +18,78 @@ #include -void AudioMixerSlaveThread::run() { - while (true) { - wait(); - - // iterate over all available nodes - SharedNodePointer node; - while (try_pop(node)) { - (this->*_function)(node); - } +void AudioMixerWorkerThread::run() { + assert(_data.numStarted < _data.numThreads); + _data.numStarted++; + bool starting = true; + while (true) { bool stopping = _stop; notify(stopping); if (stopping) { return; } + + wait(starting); + starting = false; + + if (_function) { + // iterate over all available nodes + SharedNodePointer node; + while (try_pop(node)) { + (this->*_function)(node); + } + } } } -void AudioMixerSlaveThread::wait() { +void AudioMixerWorkerThread::wait(bool starting) { + assert(_data.numStarted <= _data.numThreads); + { - Lock lock(_pool._mutex); - _pool._slaveCondition.wait(lock, [&] { - assert(_pool._numStarted <= _pool._numThreads); - return _pool._numStarted != _pool._numThreads; - }); - ++_pool._numStarted; + Lock workerLock(_data.workerMutex); + if (starting || _data.numStarted == _data.numThreads) { + do { // this is equivalent to the two-parameter "wait" call, except we're doing the test afterwards + _data.workerCondition.wait(workerLock); + } while (_data.numStarted == _data.numThreads); + } + _data.numStarted++; } - if (_pool._configure) { - _pool._configure(*this); + if (_data.configure) { + _data.configure(*this); } - _function = _pool._function; + _function = _data.function; } -void AudioMixerSlaveThread::notify(bool stopping) { - { - Lock lock(_pool._mutex); - assert(_pool._numFinished < _pool._numThreads); - ++_pool._numFinished; - if (stopping) { - ++_pool._numStopped; - } +void AudioMixerWorkerThread::notify(bool stopping) { + assert(_data.numFinished < _data.numThreads); + assert(_data.numFinished <= _data.numStarted); + int numFinished = ++_data.numFinished; + if (stopping) { + _data.numStopped++; + assert(_data.numStopped <= _data.numFinished); + } + + if (numFinished == _data.numThreads) { + Lock poolLock(_data.poolMutex); + _data.poolCondition.notify_one(); } - _pool._poolCondition.notify_one(); } -bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node) { - return _pool._queue.try_pop(node); +bool AudioMixerWorkerThread::try_pop(SharedNodePointer& node) { + return _data.queue.try_pop(node); } void AudioMixerSlavePool::processPackets(ConstIter begin, ConstIter end) { - _function = &AudioMixerSlave::processPackets; - _configure = [](AudioMixerSlave& slave) {}; + _data.function = &AudioMixerSlave::processPackets; + _data.configure = [](AudioMixerSlave& slave) {}; run(begin, end); } void AudioMixerSlavePool::mix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain) { - _function = &AudioMixerSlave::mix; - _configure = [=](AudioMixerSlave& slave) { + _data.function = &AudioMixerSlave::mix; + _data.configure = [=](AudioMixerSlave& slave) { slave.configureMix(_begin, _end, frame, numToRetain); }; @@ -89,46 +102,49 @@ void AudioMixerSlavePool::run(ConstIter begin, ConstIter end) { // fill the queue std::for_each(_begin, _end, [&](const SharedNodePointer& node) { - _queue.push(node); + _data.queue.push(node); }); + // run { - Lock lock(_mutex); - - // run - _numStarted = _numFinished = 0; - _slaveCondition.notify_all(); - - // wait - _poolCondition.wait(lock, [&] { - assert(_numFinished <= _numThreads); - return _numFinished == _numThreads; - }); + Lock workerLock(_data.workerMutex); + _data.numStarted = _data.numFinished = 0; + _data.workerCondition.notify_all(); + } - assert(_numStarted == _numThreads); + // wait + { + Lock poolLock(_data.poolMutex); + if (_data.numFinished < _data.numThreads) { + _data.poolCondition.wait(poolLock, [&] { + assert(_data.numFinished <= _data.numThreads); + return _data.numFinished == _data.numThreads; + }); + } } + assert(_data.numStarted == _data.numThreads); - assert(_queue.empty()); + assert(_data.queue.empty()); } void AudioMixerSlavePool::each(std::function functor) { - for (auto& slave : _slaves) { - functor(*slave.get()); + for (auto& worker : _workers) { + functor(*worker.get()); } } #ifdef DEBUG_EVENT_QUEUE void AudioMixerSlavePool::queueStats(QJsonObject& stats) { unsigned i = 0; - for (auto& slave : _slaves) { - int queueSize = ::hifi::qt::getEventQueueSize(slave.get()); + for (auto& worker : _workers) { + int queueSize = ::hifi::qt::getEventQueueSize(worker.get()); QString queueName = QString("audio_thread_event_queue_%1").arg(i); stats[queueName] = queueSize; i++; } } -#endif // DEBUG_EVENT_QUEUE +#endif // DEBUG_EVENT_QUEUE void AudioMixerSlavePool::setNumThreads(int numThreads) { // clamp to allowed size @@ -151,54 +167,76 @@ void AudioMixerSlavePool::setNumThreads(int numThreads) { } void AudioMixerSlavePool::resize(int numThreads) { - assert(_numThreads == (int)_slaves.size()); + assert(_data.numThreads == (int)_workers.size()); - qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads); + qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _data.numThreads); - Lock lock(_mutex); + if (numThreads > _data.numThreads) { + assert(_data.numFinished == _data.numThreads); - if (numThreads > _numThreads) { // start new slaves - for (int i = 0; i < numThreads - _numThreads; ++i) { - auto slave = new AudioMixerSlaveThread(*this, _workerSharedData); - QObject::connect(slave, &QThread::started, [] { setThreadName("AudioMixerSlaveThread"); }); - slave->start(); - _slaves.emplace_back(slave); + { + Lock workerLock(_data.workerMutex); + while (numThreads > _data.numThreads) { + _data.numThreads++; + auto worker = new AudioMixerWorkerThread(_data, _workerSharedData); + QObject::connect(worker, &QThread::started, [] { setThreadName("AudioMixerSlaveThread"); }); + worker->start(); + _workers.emplace_back(worker); + } + } + + // wait for the new workers to wake up and enter the wait + Lock poolLock(_data.poolMutex); + if (_data.numFinished < _data.numThreads) { + _data.poolCondition.wait(poolLock, [&] { + assert(_data.numFinished <= _data.numThreads); + return _data.numFinished == _data.numThreads; + }); } - } else if (numThreads < _numThreads) { - auto extraBegin = _slaves.begin() + numThreads; + } else if (numThreads < _data.numThreads) { + auto extraBegin = _workers.begin() + numThreads; // mark slaves to stop... - auto slave = extraBegin; - while (slave != _slaves.end()) { - (*slave)->_stop = true; - ++slave; + auto worker = extraBegin; + while (worker != _workers.end()) { + (*worker)->stop(); + ++worker; } // ...cycle them until they do stop... - _numStopped = 0; - while (_numStopped != (_numThreads - numThreads)) { - _numStarted = _numFinished = _numStopped; - _slaveCondition.notify_all(); - _poolCondition.wait(lock, [&] { - assert(_numFinished <= _numThreads); - return _numFinished == _numThreads; - }); + _data.numStopped = 0; + while (_data.numStopped != (_data.numThreads - numThreads)) { + { + Lock workerLock(_data.workerMutex); + _data.numStarted = _data.numFinished = 0; + _data.workerCondition.notify_all(); + } + + Lock poolLock(_data.poolMutex); + if (_data.numFinished < _data.numThreads) { + _data.poolCondition.wait(poolLock, [&] { + assert(_data.numFinished <= _data.numThreads); + return _data.numFinished == _data.numThreads; + }); + } + assert(_data.numStopped == (_data.numThreads - numThreads)); } // ...wait for threads to finish... - slave = extraBegin; - while (slave != _slaves.end()) { - QThread* thread = reinterpret_cast(slave->get()); + worker = extraBegin; + while (worker != _workers.end()) { + QThread* thread = reinterpret_cast(worker->get()); static const int MAX_THREAD_WAIT_TIME = 10; thread->wait(MAX_THREAD_WAIT_TIME); - ++slave; + ++worker; } // ...and erase them - _slaves.erase(extraBegin, _slaves.end()); + _workers.erase(extraBegin, _workers.end()); } - _numThreads = _numStarted = _numFinished = numThreads; - assert(_numThreads == (int)_slaves.size()); + _data.numThreads = _data.numStarted = _data.numFinished = numThreads; + assert(_data.numThreads == (int)_workers.size()); + qDebug("%s: completed", __FUNCTION__); } diff --git a/assignment-client/src/audio/AudioMixerSlavePool.h b/assignment-client/src/audio/AudioMixerSlavePool.h index 3807db05410..d0de2734b5b 100644 --- a/assignment-client/src/audio/AudioMixerSlavePool.h +++ b/assignment-client/src/audio/AudioMixerSlavePool.h @@ -17,44 +17,80 @@ #include #include -#include +#include // for DEBUG_EVENT_QUEUE #include #include "AudioMixerSlave.h" -class AudioMixerSlavePool; +// Private worker pool data that is shared and accessible with the worker threads. This describes +// what information is needs to be thread-safe +struct AudioMixerWorkerPoolData { + using Queue = tbb::concurrent_queue; + using Mutex = std::mutex; + using ConditionVariable = std::condition_variable; + + // synchronization state + Mutex poolMutex; // only used for _poolCondition at the moment + ConditionVariable poolCondition; // woken when work has been completed (_numStarted = _numFinished = _numThreads) + Mutex workerMutex; // only used for _workerCondition at the moment + ConditionVariable workerCondition; // woken when work needs to be done (_numStarted < _numThreads) + + // The variables below this point are alternately owned by the pool or by the workers collectively. + // When idle they are owned by the pool. + // Moving ownership to the workers is done by setting _numStarted = _numFinished = 0 and waking _workerCondition + // Moving ownership to the pool is done when _numFinished == _numThreads and is done by waking _poolCondition + + void (AudioMixerSlave::*function)(const SharedNodePointer& node){ nullptr }; // r/o when owned by workers, r/w when owned by pool + std::function configure{ nullptr }; // r/o when owned by workers, r/w when owned by pool + + // Number of currently-running worker threads + // r/o when owned by workersves, r/w when owned by pool + int numThreads{ 0 }; + + // Number of worker threads "awake" and processing the current request (0 <= _numStarted <= _numThreads) + // incremented when owned by workers, r/w when owned by pool + std::atomic numStarted{ 0 }; + + // Number of worker threads finished with the current request (0 <= _numFinished <= _numStarted) + // incremented when owned by workers, r/w when owned by pool + std::atomic numFinished{ 0 }; -class AudioMixerSlaveThread : public QThread, public AudioMixerSlave { + // Number of worker threads shutting down when asked to (0 <= _numStopped <= _numFinished) + // incremented when owned by workers, r/w when owned by pool + std::atomic numStopped{ 0 }; + + // frame state + Queue queue; +}; + +class AudioMixerWorkerThread : public QThread, public AudioMixerSlave { Q_OBJECT using ConstIter = NodeList::const_iterator; using Mutex = std::mutex; using Lock = std::unique_lock; public: - AudioMixerSlaveThread(AudioMixerSlavePool& pool, AudioMixerSlave::SharedData& sharedData) - : AudioMixerSlave(sharedData), _pool(pool) {} + AudioMixerWorkerThread(AudioMixerWorkerPoolData& data, AudioMixerSlave::SharedData& sharedData) : + AudioMixerSlave(sharedData), _data(data) {} void run() override final; + inline void stop() { _stop = true; } private: - friend class AudioMixerSlavePool; - - void wait(); + void wait(bool starting); void notify(bool stopping); bool try_pop(SharedNodePointer& node); - AudioMixerSlavePool& _pool; + AudioMixerWorkerPoolData& _data; void (AudioMixerSlave::*_function)(const SharedNodePointer& node) { nullptr }; - bool _stop { false }; + volatile bool _stop { false }; // using volatile here mostly for compiler hinting, recognize it has minimal meaning }; -// Slave pool for audio mixers +// Worker pool for audio mixers // AudioMixerSlavePool is not thread-safe! It should be instantiated and used from a single thread. class AudioMixerSlavePool { - using Queue = tbb::concurrent_queue; using Mutex = std::mutex; using Lock = std::unique_lock; - using ConditionVariable = std::condition_variable; public: using ConstIter = NodeList::const_iterator; @@ -63,13 +99,13 @@ class AudioMixerSlavePool { : _workerSharedData(sharedData) { setNumThreads(numThreads); } ~AudioMixerSlavePool() { resize(0); } - // process packets on slave threads + // process packets on worker threads void processPackets(ConstIter begin, ConstIter end); - // mix on slave threads + // mix on worker threads void mix(ConstIter begin, ConstIter end, unsigned int frame, int numToRetain); - // iterate over all slaves + // iterate over all workers void each(std::function functor); #ifdef DEBUG_EVENT_QUEUE @@ -77,35 +113,20 @@ class AudioMixerSlavePool { #endif void setNumThreads(int numThreads); - int numThreads() { return _numThreads; } + int numThreads() { return _data.numThreads; } private: void run(ConstIter begin, ConstIter end); void resize(int numThreads); - std::vector> _slaves; - - friend void AudioMixerSlaveThread::wait(); - friend void AudioMixerSlaveThread::notify(bool stopping); - friend bool AudioMixerSlaveThread::try_pop(SharedNodePointer& node); - - // synchronization state - Mutex _mutex; - ConditionVariable _slaveCondition; - ConditionVariable _poolCondition; - void (AudioMixerSlave::*_function)(const SharedNodePointer& node); - std::function _configure; - int _numThreads { 0 }; - int _numStarted { 0 }; // guarded by _mutex - int _numFinished { 0 }; // guarded by _mutex - int _numStopped { 0 }; // guarded by _mutex + std::vector> _workers; // frame state - Queue _queue; ConstIter _begin; ConstIter _end; AudioMixerSlave::SharedData& _workerSharedData; + AudioMixerWorkerPoolData _data; }; #endif // hifi_AudioMixerSlavePool_h diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp index 027e68e88b0..a1413b77a60 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.cpp @@ -14,71 +14,84 @@ #include #include -void AvatarMixerSlaveThread::run() { - while (true) { - wait(); - - // iterate over all available nodes - SharedNodePointer node; - while (try_pop(node)) { - (this->*_function)(node); - } +void AvatarMixerWorkerThread::run() { + assert(_data.numStarted < _data.numThreads); + _data.numStarted++; + bool starting = true; + while (true) { bool stopping = _stop; notify(stopping); if (stopping) { return; } + + wait(starting); + starting = false; + + if (_function) { + // iterate over all available nodes + SharedNodePointer node; + while (try_pop(node)) { + (this->*_function)(node); + } + } } } -void AvatarMixerSlaveThread::wait() { +void AvatarMixerWorkerThread::wait(bool starting) { + assert(_data.numStarted <= _data.numThreads); + { - Lock lock(_pool._mutex); - _pool._slaveCondition.wait(lock, [&] { - assert(_pool._numStarted <= _pool._numThreads); - return _pool._numStarted != _pool._numThreads; - }); - ++_pool._numStarted; + Lock workerLock(_data.workerMutex); + if (starting || _data.numStarted == _data.numThreads) { + do { // this is equivalent to the two-parameter "wait" call, except we're doing the test afterwards + _data.workerCondition.wait(workerLock); + } while (_data.numStarted == _data.numThreads); + } + _data.numStarted++; } - if (_pool._configure) { - _pool._configure(*this); + + if (_data.configure) { + _data.configure(*this); } - _function = _pool._function; + _function = _data.function; } -void AvatarMixerSlaveThread::notify(bool stopping) { - { - Lock lock(_pool._mutex); - assert(_pool._numFinished < _pool._numThreads); - ++_pool._numFinished; - if (stopping) { - ++_pool._numStopped; - } +void AvatarMixerWorkerThread::notify(bool stopping) { + assert(_data.numFinished < _data.numThreads); + assert(_data.numFinished <= _data.numStarted); + int numFinished = ++_data.numFinished; + if (stopping) { + _data.numStopped++; + assert(_data.numStopped <= _data.numFinished); + } + + if (numFinished == _data.numThreads) { + Lock poolLock(_data.poolMutex); + _data.poolCondition.notify_one(); } - _pool._poolCondition.notify_one(); } -bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node) { - return _pool._queue.try_pop(node); +bool AvatarMixerWorkerThread::try_pop(SharedNodePointer& node) { + return _data.queue.try_pop(node); } void AvatarMixerSlavePool::processIncomingPackets(ConstIter begin, ConstIter end) { - _function = &AvatarMixerSlave::processIncomingPackets; - _configure = [=](AvatarMixerSlave& slave) { + _data.function = &AvatarMixerSlave::processIncomingPackets; + _data.configure = [=](AvatarMixerSlave& slave) { slave.configure(begin, end); }; run(begin, end); } -void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, +void AvatarMixerSlavePool::broadcastAvatarData(ConstIter begin, ConstIter end, p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode, float throttlingRatio) { - _function = &AvatarMixerSlave::broadcastAvatarData; - _configure = [=](AvatarMixerSlave& slave) { - slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode, throttlingRatio, - _priorityReservedFraction); - }; + _data.function = &AvatarMixerSlave::broadcastAvatarData; + _data.configure = [=](AvatarMixerSlave& slave) { + slave.configureBroadcast(begin, end, lastFrameTimestamp, maxKbpsPerNode, throttlingRatio, _priorityReservedFraction); + }; run(begin, end); } @@ -88,47 +101,49 @@ void AvatarMixerSlavePool::run(ConstIter begin, ConstIter end) { // fill the queue std::for_each(_begin, _end, [&](const SharedNodePointer& node) { - _queue.push(node); + _data.queue.push(node); }); + // run { - Lock lock(_mutex); - - // run - _numStarted = _numFinished = 0; - _slaveCondition.notify_all(); - - // wait - _poolCondition.wait(lock, [&] { - assert(_numFinished <= _numThreads); - return _numFinished == _numThreads; - }); + Lock workerLock(_data.workerMutex); + _data.numStarted = _data.numFinished = 0; + _data.workerCondition.notify_all(); + } - assert(_numStarted == _numThreads); + // wait + { + Lock poolLock(_data.poolMutex); + if (_data.numFinished < _data.numThreads) { + _data.poolCondition.wait(poolLock, [&] { + assert(_data.numFinished <= _data.numThreads); + return _data.numFinished == _data.numThreads; + }); + } } + assert(_data.numStarted == _data.numThreads); - assert(_queue.empty()); + assert(_data.queue.empty()); } - void AvatarMixerSlavePool::each(std::function functor) { - for (auto& slave : _slaves) { - functor(*slave.get()); + for (auto& worker : _workers) { + functor(*worker.get()); } } #ifdef DEBUG_EVENT_QUEUE void AvatarMixerSlavePool::queueStats(QJsonObject& stats) { unsigned i = 0; - for (auto& slave : _slaves) { - int queueSize = ::hifi::qt::getEventQueueSize(slave.get()); + for (auto& worker : _workers) { + int queueSize = ::hifi::qt::getEventQueueSize(worker.get()); QString queueName = QString("avatar_thread_event_queue_%1").arg(i); stats[queueName] = queueSize; i++; } } -#endif // DEBUG_EVENT_QUEUE +#endif // DEBUG_EVENT_QUEUE void AvatarMixerSlavePool::setNumThreads(int numThreads) { // clamp to allowed size @@ -151,53 +166,77 @@ void AvatarMixerSlavePool::setNumThreads(int numThreads) { } void AvatarMixerSlavePool::resize(int numThreads) { - assert(_numThreads == (int)_slaves.size()); + assert(_data.numThreads == (int)_workers.size()); - qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _numThreads); + qDebug("%s: set %d threads (was %d)", __FUNCTION__, numThreads, _data.numThreads); - Lock lock(_mutex); + if (numThreads > _data.numThreads) { + assert(_data.numFinished == _data.numThreads); - if (numThreads > _numThreads) { // start new slaves - for (int i = 0; i < numThreads - _numThreads; ++i) { - auto slave = new AvatarMixerSlaveThread(*this, _slaveSharedData); - slave->start(); - _slaves.emplace_back(slave); + { + Lock workerLock(_data.workerMutex); + while (numThreads > _data.numThreads) { + _data.numThreads++; + auto worker = new AvatarMixerWorkerThread(_data, _slaveSharedData); + worker->start(); + _workers.emplace_back(worker); + } + } + + // wait for the new workers to wake up and enter the wait + Lock poolLock(_data.poolMutex); + if (_data.numFinished < _data.numThreads) { + _data.poolCondition.wait(poolLock, [&] { + assert(_data.numFinished <= _data.numThreads); + return _data.numFinished == _data.numThreads; + }); } - } else if (numThreads < _numThreads) { - auto extraBegin = _slaves.begin() + numThreads; + } else if (numThreads < _data.numThreads) { + auto extraBegin = _workers.begin() + numThreads; // mark slaves to stop... - auto slave = extraBegin; - while (slave != _slaves.end()) { - (*slave)->_stop = true; - ++slave; + auto worker = extraBegin; + while (worker != _workers.end()) { + (*worker)->stop(); + ++worker; } // ...cycle them until they do stop... - _numStopped = 0; - while (_numStopped != (_numThreads - numThreads)) { - _numStarted = _numFinished = _numStopped; - _slaveCondition.notify_all(); - _poolCondition.wait(lock, [&] { - assert(_numFinished <= _numThreads); - return _numFinished == _numThreads; - }); + _data.numStopped = 0; + while (_data.numStopped != (_data.numThreads - numThreads)) { + { + Lock workerLock(_data.workerMutex); + _data.numStarted = _data.numFinished = 0; + _data.workerCondition.notify_all(); + } + + { + Lock poolLock(_data.poolMutex); + if (_data.numFinished < _data.numThreads) { + _data.poolCondition.wait(poolLock, [&] { + assert(_data.numFinished <= _data.numThreads); + return _data.numFinished == _data.numThreads; + }); + } + } + assert(_data.numStopped == (_data.numThreads - numThreads)); } // ...wait for threads to finish... - slave = extraBegin; - while (slave != _slaves.end()) { - QThread* thread = reinterpret_cast(slave->get()); + worker = extraBegin; + while (worker != _workers.end()) { + QThread* thread = reinterpret_cast(worker->get()); static const int MAX_THREAD_WAIT_TIME = 10; thread->wait(MAX_THREAD_WAIT_TIME); - ++slave; + ++worker; } // ...and erase them - _slaves.erase(extraBegin, _slaves.end()); + _workers.erase(extraBegin, _workers.end()); } - _numThreads = _numStarted = _numFinished = numThreads; - assert(_numThreads == (int)_slaves.size()); + _data.numThreads = _data.numStarted = _data.numFinished = numThreads; + assert(_data.numThreads == (int)_workers.size()); + qDebug("%s: completed", __FUNCTION__); } diff --git a/assignment-client/src/avatars/AvatarMixerSlavePool.h b/assignment-client/src/avatars/AvatarMixerSlavePool.h index 915c6d8dc44..5e4d436a081 100644 --- a/assignment-client/src/avatars/AvatarMixerSlavePool.h +++ b/assignment-client/src/avatars/AvatarMixerSlavePool.h @@ -20,44 +20,80 @@ #include #include -#include +#include // for DEBUG_EVENT_QUEUE #include "AvatarMixerSlave.h" -class AvatarMixerSlavePool; +// Private worker pool data that is shared and accessible with the worker threads. This describes +// what information is needs to be thread-safe +struct AvatarMixerWorkerPoolData { + using Queue = tbb::concurrent_queue; + using Mutex = std::mutex; + using ConditionVariable = std::condition_variable; + + // synchronization state + Mutex poolMutex; // only used for _poolCondition at the moment + ConditionVariable poolCondition; // woken when work has been completed (_numStarted = _numFinished = _numThreads) + Mutex workerMutex; // only used for _workerCondition at the moment + ConditionVariable workerCondition; // woken when work needs to be done (_numStarted < _numThreads) + + // The variables below this point are alternately owned by the pool or by the workers collectively. + // When idle they are owned by the pool. + // Moving ownership to the workers is done by setting _numStarted = _numFinished = 0 and waking _slaveCondition + // Moving ownership to the pool is done when _numFinished == _numThreads and is done by waking _poolCondition + + void (AvatarMixerSlave::*function)(const SharedNodePointer& node){ nullptr }; // r/o when owned by workers, r/w when owned by pool + std::function configure{ nullptr }; // r/o when owned by workers, r/w when owned by pool + + // Number of currently-running worker threads + // r/o when owned by workers, r/w when owned by pool + int numThreads{ 0 }; + + // Number of worker threads "awake" and processing the current request (0 <= _numStarted <= _numThreads) + // incremented when owned by workers, r/w when owned by pool + std::atomic numStarted{ 0 }; + + // Number of worker threads finished with the current request (0 <= _numStarted <= _numThreads) + // incremented when owned by workers, r/w when owned by pool + std::atomic numFinished{ 0 }; + + // Number of worker threads shutting down when asked to (0 <= _numStarted <= _numThreads) + // incremented when owned by workers, r/w when owned by pool + std::atomic numStopped{ 0 }; -class AvatarMixerSlaveThread : public QThread, public AvatarMixerSlave { + // frame state + Queue queue; +}; + +class AvatarMixerWorkerThread : public QThread, public AvatarMixerSlave { Q_OBJECT using ConstIter = NodeList::const_iterator; using Mutex = std::mutex; using Lock = std::unique_lock; public: - AvatarMixerSlaveThread(AvatarMixerSlavePool& pool, SlaveSharedData* slaveSharedData) : - AvatarMixerSlave(slaveSharedData), _pool(pool) {}; + AvatarMixerWorkerThread(AvatarMixerWorkerPoolData& data, SlaveSharedData* slaveSharedData) : + AvatarMixerSlave(slaveSharedData), _data(data){}; void run() override final; + inline void stop() { _stop = true; } private: - friend class AvatarMixerSlavePool; - - void wait(); + void wait(bool starting); void notify(bool stopping); bool try_pop(SharedNodePointer& node); - AvatarMixerSlavePool& _pool; + AvatarMixerWorkerPoolData& _data; void (AvatarMixerSlave::*_function)(const SharedNodePointer& node) { nullptr }; - bool _stop { false }; + volatile bool _stop{ false }; // using volatile here mostly for compiler hinting, recognize it has minimal meaning }; -// Slave pool for avatar mixers +// Worker pool for avatar mixers // AvatarMixerSlavePool is not thread-safe! It should be instantiated and used from a single thread. class AvatarMixerSlavePool { - using Queue = tbb::concurrent_queue; using Mutex = std::mutex; using Lock = std::unique_lock; - using ConditionVariable = std::condition_variable; public: using ConstIter = NodeList::const_iterator; @@ -66,12 +102,12 @@ class AvatarMixerSlavePool { _slaveSharedData(slaveSharedData) { setNumThreads(numThreads); } ~AvatarMixerSlavePool() { resize(0); } - // Jobs the slave pool can do... + // Jobs the worker pool can do... void processIncomingPackets(ConstIter begin, ConstIter end); void broadcastAvatarData(ConstIter begin, ConstIter end, p_high_resolution_clock::time_point lastFrameTimestamp, float maxKbpsPerNode, float throttlingRatio); - // iterate over all slaves + // iterate over all workers void each(std::function functor); #ifdef DEBUG_EVENT_QUEUE @@ -79,7 +115,7 @@ class AvatarMixerSlavePool { #endif void setNumThreads(int numThreads); - int numThreads() const { return _numThreads; } + int numThreads() const { return _data.numThreads; } void setPriorityReservedFraction(float fraction) { _priorityReservedFraction = fraction; } float getPriorityReservedFraction() const { return _priorityReservedFraction; } @@ -88,33 +124,17 @@ class AvatarMixerSlavePool { void run(ConstIter begin, ConstIter end); void resize(int numThreads); - std::vector> _slaves; - - friend void AvatarMixerSlaveThread::wait(); - friend void AvatarMixerSlaveThread::notify(bool stopping); - friend bool AvatarMixerSlaveThread::try_pop(SharedNodePointer& node); - - // synchronization state - Mutex _mutex; - ConditionVariable _slaveCondition; - ConditionVariable _poolCondition; - void (AvatarMixerSlave::*_function)(const SharedNodePointer& node); - std::function _configure; + std::vector> _workers; // Set from Domain Settings: float _priorityReservedFraction { 0.4f }; - int _numThreads { 0 }; - - int _numStarted { 0 }; // guarded by _mutex - int _numFinished { 0 }; // guarded by _mutex - int _numStopped { 0 }; // guarded by _mutex // frame state - Queue _queue; ConstIter _begin; ConstIter _end; SlaveSharedData* _slaveSharedData; + AvatarMixerWorkerPoolData _data; }; #endif // hifi_AvatarMixerSlavePool_h