Skip to content

Commit 73be52b

Browse files
AntreesyAntreesy
committed
feat: implement NoiseSuppressor node for audio processing
Signed-off-by: Maksim Sukharev <antreesy.web@gmail.com>
1 parent 7371723 commit 73be52b

5 files changed

Lines changed: 186 additions & 3 deletions

File tree

‎src/components/MediaSettings/AdvancedAudioDialog.vue‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import NcFormBox from '@nextcloud/vue/components/NcFormBox'
1010
import NcFormBoxSwitch from '@nextcloud/vue/components/NcFormBoxSwitch'
1111
import { useDevices } from '../../composables/useDevices.js'
1212
import { useSettingsStore } from '../../stores/settings.ts'
13+
import { localMediaModel } from '../../utils/webrtc/index.js'
1314
1415
const props = defineProps<{
1516
container?: string
@@ -66,6 +67,14 @@ function onClosing(result?: unknown) {
6667
)) {
6768
// Apply changes to audio stream
6869
updateAudioStream(true)
70+
71+
if (localMediaModel.getWebRtc()) {
72+
if (settingsStore.noiseSuppressionWithModel) {
73+
localMediaModel.enableNoiseSuppression()
74+
} else {
75+
localMediaModel.disableNoiseSuppression()
76+
}
77+
}
6978
}
7079
7180
emit('close', result)

‎src/utils/media/pipeline/NoiseSuppressor.js‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/**
2+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
3+
* SPDX-License-Identifier: AGPL-3.0-or-later
4+
*/
5+
6+
import {
7+
destroyNoiseSuppressionWorklet,
8+
processNoiseSuppression,
9+
registerNoiseSuppressionWorklet,
10+
} from '../../suppressNoise.ts'
11+
import TrackSinkSource from './TrackSinkSource.js'
12+
13+
/**
14+
* Processor node to enable or disable noise suppression.
15+
*
16+
* A single input track slot with the default id is accepted. A single output
17+
* track slot with the default id is provided.
18+
*
19+
* The track can be enabled and disabled by calling "setEnabled(bool)".
20+
*
21+
* When enabled, the node processes the input audio track to suppress noise and
22+
* outputs a new, processed track. When disabled, it passes the original audio
23+
* track through without modification.
24+
*
25+
* The node manages the lifecycle of the underlying noise suppression worklet,
26+
* registering a consumer when enabled and destroying it when disabled.
27+
*
28+
* -----------------
29+
* | |
30+
* ---> | NoiseSuppressor | --->
31+
* | |
32+
* -----------------
33+
*/
34+
export default class NoiseSuppressor extends TrackSinkSource {
35+
constructor() {
36+
super()
37+
38+
this._addInputTrackSlot()
39+
this._addOutputTrackSlot()
40+
41+
/** Whether to send processed or unmodified stream to the output */
42+
this._enabled = false
43+
/** Whether the current input audio track is enabled */
44+
this._audioEnabled = false
45+
/** Timeout for worklets destroying and garbage collection (in case of reconnect) */
46+
this._stopTimer = null
47+
/** Unique consumer symbol to track worklet subscribers */
48+
this._noiseSuppressionConsumer = null
49+
/** Pending registration promise to keep _startEffect synchronous */
50+
this._noiseSuppressionRegistrationPromise = null
51+
}
52+
53+
setEnabled(enabled) {
54+
if (this._enabled === enabled) {
55+
return
56+
}
57+
58+
this._enabled = enabled
59+
60+
if (enabled) {
61+
this._startEffect()
62+
} else {
63+
this._stopEffect()
64+
}
65+
}
66+
67+
_handleInputTrack(trackId, track) {
68+
if (this._enabled) {
69+
this._startEffect()
70+
} else {
71+
this._stopEffect()
72+
}
73+
}
74+
75+
_handleInputTrackEnabled(trackId, enabled) {
76+
this._audioEnabled = enabled
77+
this._setOutputTrackEnabled('default', enabled)
78+
}
79+
80+
_startEffect() {
81+
if (this._stopTimer) {
82+
clearTimeout(this._stopTimer)
83+
this._stopTimer = null
84+
}
85+
86+
const track = this.getInputTrack()
87+
this._audioEnabled = track?.enabled ?? false
88+
if (!track) {
89+
this._setOutputTrack('default', track)
90+
return
91+
}
92+
93+
if (!this._noiseSuppressionConsumer) {
94+
// Start initializing the worklet if not already in progress. While waiting, set default track as an output.
95+
if (!this._noiseSuppressionRegistrationPromise) {
96+
this._noiseSuppressionRegistrationPromise = registerNoiseSuppressionWorklet()
97+
.then((consumer) => {
98+
this._noiseSuppressionRegistrationPromise = null
99+
this._noiseSuppressionConsumer = consumer
100+
101+
if (this._enabled) {
102+
this._startEffect()
103+
} else {
104+
this._stopEffect()
105+
}
106+
})
107+
.catch((error) => {
108+
this._noiseSuppressionRegistrationPromise = null
109+
console.error(error)
110+
})
111+
}
112+
113+
this._setOutputTrack('default', track)
114+
return
115+
}
116+
117+
const inputStream = new MediaStream([track])
118+
const processedStream = processNoiseSuppression(inputStream, true)
119+
const processedTrack = processedStream.getAudioTracks()[0]
120+
processedTrack.enabled = this._audioEnabled
121+
122+
this._setOutputTrack('default', processedTrack)
123+
}
124+
125+
_stopEffect() {
126+
if (!this._stopTimer) {
127+
this._stopTimer = setTimeout(async () => {
128+
if (this._noiseSuppressionConsumer) {
129+
await destroyNoiseSuppressionWorklet(this._noiseSuppressionConsumer)
130+
this._noiseSuppressionConsumer = null
131+
}
132+
this._stopTimer = null
133+
}, 3_000)
134+
}
135+
136+
const track = this.getInputTrack()
137+
this._setOutputTrack('default', track)
138+
}
139+
}

‎src/utils/webrtc/index.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ async function signalingJoinCall(token, flags, silent, recordingConsent, silentF
262262

263263
// The previous state might be wiped after the media is started, so
264264
// it should be saved now.
265+
const noiseSuppressionWithModel = BrowserStorage.getItem('noiseSuppressionWithModel') === 'true'
265266
const enableAudio = !BrowserStorage.getItem('audioDisabled_' + token)
266267
const enableVideo = !BrowserStorage.getItem('videoDisabled_' + token)
267268
const enableVirtualBackground = !!BrowserStorage.getItem('virtualBackgroundEnabled')
@@ -286,6 +287,11 @@ async function signalingJoinCall(token, flags, silent, recordingConsent, silentF
286287
} else {
287288
localMediaModel.disableVirtualBackground()
288289
}
290+
if (noiseSuppressionWithModel) {
291+
localMediaModel.enableNoiseSuppression()
292+
} else {
293+
localMediaModel.disableNoiseSuppression()
294+
}
289295
if (virtualBackgroundType === VIRTUAL_BACKGROUND.BACKGROUND_TYPE.IMAGE) {
290296
localMediaModel.setVirtualBackgroundImage(virtualBackgroundUrl)
291297
} else if (virtualBackgroundType === VIRTUAL_BACKGROUND.BACKGROUND_TYPE.VIDEO) {

‎src/utils/webrtc/models/LocalMediaModel.js‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,20 @@ LocalMediaModel.prototype = {
377377
this._webRtc.pauseVideo()
378378
},
379379

380+
enableNoiseSuppression() {
381+
if (!this._webRtc) {
382+
throw new Error('WebRtc not initialized yet')
383+
}
384+
this._webRtc.webrtc.enableNoiseSuppression()
385+
},
386+
387+
disableNoiseSuppression() {
388+
if (!this._webRtc) {
389+
throw new Error('WebRtc not initialized yet')
390+
}
391+
this._webRtc.webrtc.disableNoiseSuppression()
392+
},
393+
380394
enableVirtualBackground() {
381395
if (!this._webRtc) {
382396
throw new Error('WebRtc not initialized yet')

‎src/utils/webrtc/simplewebrtc/localmedia.js‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import util from 'util'
88
import WildEmitter from 'wildemitter'
99
import BlackVideoEnforcer from '../../media/pipeline/BlackVideoEnforcer.js'
1010
import MediaDevicesSource from '../../media/pipeline/MediaDevicesSource.js'
11+
import NoiseSuppressor from '../../media/pipeline/NoiseSuppressor.js'
1112
import SpeakingMonitor from '../../media/pipeline/SpeakingMonitor.js'
1213
import TrackConstrainer from '../../media/pipeline/TrackConstrainer.js'
1314
import TrackEnabler from '../../media/pipeline/TrackEnabler.js'
@@ -55,6 +56,8 @@ export default function LocalMedia(opts) {
5556
this._audioTrackEnabler = new TrackEnabler()
5657
this._videoTrackEnabler = new TrackEnabler()
5758

59+
this._noiseSuppressor = new NoiseSuppressor()
60+
5861
this._videoTrackConstrainer = new TrackConstrainer()
5962

6063
this._virtualBackground = new VirtualBackground()
@@ -99,9 +102,11 @@ export default function LocalMedia(opts) {
99102
this._mediaDevicesSource.connectTrackSink('audio', this._audioTrackEnabler)
100103
this._mediaDevicesSource.connectTrackSink('video', this._videoTrackEnabler)
101104

102-
this._audioTrackEnabler.connectTrackSink('default', this._speakingMonitor)
103-
this._audioTrackEnabler.connectTrackSink('default', this._trackToStream, 'audio')
104-
this._audioTrackEnabler.connectTrackSink('default', this._trackToSentStream, 'audio')
105+
this._audioTrackEnabler.connectTrackSink('default', this._noiseSuppressor)
106+
107+
this._noiseSuppressor.connectTrackSink('default', this._speakingMonitor)
108+
this._noiseSuppressor.connectTrackSink('default', this._trackToStream, 'audio')
109+
this._noiseSuppressor.connectTrackSink('default', this._trackToSentStream, 'audio')
105110

106111
this._videoTrackEnabler.connectTrackSink('default', this._videoTrackConstrainer)
107112

@@ -392,6 +397,16 @@ LocalMedia.prototype.resumeVideo = function() {
392397
this.emit('videoOn')
393398
}
394399

400+
LocalMedia.prototype.enableNoiseSuppression = function() {
401+
this._noiseSuppressor.setEnabled(true)
402+
this.emit('noiseSuppressionOn')
403+
}
404+
405+
LocalMedia.prototype.disableNoiseSuppression = function() {
406+
this._noiseSuppressor.setEnabled(false)
407+
this.emit('noiseSuppressionOff')
408+
}
409+
395410
LocalMedia.prototype.enableVirtualBackground = function() {
396411
this._virtualBackground.setEnabled(true)
397412
this.emit('virtualBackgroundOn')

0 commit comments

Comments
 (0)