|
| 1 | +/* |
| 2 | + * Nextcloud Talk - Android Client |
| 3 | + * |
| 4 | + * SPDX-FileCopyrightText: 2025 Julius Linus <juliuslinus1@gmail.com> |
| 5 | + * SPDX-License-Identifier: GPL-3.0-or-later |
| 6 | + */ |
| 7 | + |
| 8 | +package com.nextcloud.talk.camera |
| 9 | + |
| 10 | +import android.content.Context |
| 11 | +import android.os.Handler |
| 12 | +import android.os.HandlerThread |
| 13 | +import android.util.Log |
| 14 | +import android.util.LruCache |
| 15 | +import io.github.crow_misia.libyuv.AbgrBuffer |
| 16 | +import io.github.crow_misia.libyuv.I420Buffer |
| 17 | +import io.github.crow_misia.libyuv.PlanePrimitive |
| 18 | +import org.webrtc.JavaI420Buffer |
| 19 | +import org.webrtc.SurfaceTextureHelper |
| 20 | +import org.webrtc.VideoFrame |
| 21 | +import org.webrtc.VideoProcessor |
| 22 | +import org.webrtc.VideoSink |
| 23 | +import org.webrtc.YuvHelper |
| 24 | +import java.nio.ByteBuffer |
| 25 | + |
| 26 | +@Suppress("TooGenericExceptionCaught") |
| 27 | +class BackgroundBlurFrameProcessor(val context: Context, val surfaceTextureHelper: SurfaceTextureHelper) : |
| 28 | + VideoProcessor, |
| 29 | + ImageSegmenterHelper.SegmenterListener { |
| 30 | + |
| 31 | + companion object { |
| 32 | + val TAG: String = this::class.java.simpleName |
| 33 | + const val GPU_THREAD: String = "BackgroundBlur" |
| 34 | + const val FLOAT_ROTATION = 180.0f |
| 35 | + const val INT_4 = 4 |
| 36 | + const val MAX_NUM_FRAMES = 10 |
| 37 | + } |
| 38 | + |
| 39 | + private var sink: VideoSink? = null |
| 40 | + private var segmenterHelper: ImageSegmenterHelper? = null |
| 41 | + private var backgroundBlurGPUProcessor: BackgroundBlurGPUProcessor? = null |
| 42 | + |
| 43 | + /* This is to hold meta information between MediaPipe and GPU Render threads, in a thread safe way |
| 44 | + A LRU (least recently used) cache, holds up to MAX_NUM_FRAMES, before evicting the least recently used |
| 45 | + if full. This is used because in case an error occurs with MediaPipe, and a frame is dropped, the frame might stay |
| 46 | + in the map indefinitely, unable to be cleaned up by the garbage collector, therefore causing a memory leak. With the |
| 47 | + LRU Cache, the frame would end up being cleaned eventually as the program runs */ |
| 48 | + private var rotationMap = LruCache<Long, Float>(MAX_NUM_FRAMES) |
| 49 | + private val frameBufferMap = LruCache<Long, ByteBuffer>(MAX_NUM_FRAMES) |
| 50 | + |
| 51 | + // Dedicated Thread for OpenGL Operations |
| 52 | + private var glThread: HandlerThread? = null |
| 53 | + private var glHandler: Handler? = null |
| 54 | + |
| 55 | + // SegmentationListener Interface |
| 56 | + |
| 57 | + override fun onError(error: String, errorCode: Int) { |
| 58 | + Log.e(TAG, "Error $errorCode: $error") |
| 59 | + } |
| 60 | + |
| 61 | + override fun onResults(resultBundle: ImageSegmenterHelper.ResultBundle) { |
| 62 | + val rotation = rotationMap[resultBundle.inferenceTime] ?: 0f |
| 63 | + val frameBuffer = frameBufferMap[resultBundle.inferenceTime] |
| 64 | + |
| 65 | + // Remove once used to prevent mem leaks |
| 66 | + rotationMap.synchronizedRemove(resultBundle.inferenceTime) |
| 67 | + frameBufferMap.synchronizedRemove(resultBundle.inferenceTime) |
| 68 | + |
| 69 | + if (frameBuffer == null) { |
| 70 | + Log.e(TAG, "Critical Error in onResults: FrameBufferMap[${resultBundle.inferenceTime}] was null") |
| 71 | + return |
| 72 | + } |
| 73 | + |
| 74 | + glHandler?.post { |
| 75 | + // This block runs safely on gpu thread |
| 76 | + backgroundBlurGPUProcessor?.let { scaler -> |
| 77 | + try { |
| 78 | + val drawArray = scaler.process( |
| 79 | + resultBundle.mask, |
| 80 | + frameBuffer, |
| 81 | + resultBundle.width, |
| 82 | + resultBundle.height, |
| 83 | + rotation |
| 84 | + ) |
| 85 | + |
| 86 | + val webRTCBuffer = drawArray.convertToWebRTCBuffer(resultBundle.width, resultBundle.height) |
| 87 | + val videoFrame = VideoFrame(webRTCBuffer, 0, resultBundle.inferenceTime) |
| 88 | + |
| 89 | + // This should run on the CaptureThread |
| 90 | + surfaceTextureHelper.handler.post { |
| 91 | + Log.d(TAG, "Sent VideoFrame to sink on :${Thread.currentThread().name}") |
| 92 | + sink?.onFrame(videoFrame) |
| 93 | + |
| 94 | + // webRTCBuffer usually needs release() if it's not a JavaI420Buffer wrapper that auto-GCs, |
| 95 | + // but JavaI420Buffer.wrap() relies on GC. |
| 96 | + videoFrame.release() |
| 97 | + } |
| 98 | + } catch (e: Exception) { |
| 99 | + Log.e(TAG, "Error processing frame on GL Thread", e) |
| 100 | + } |
| 101 | + } |
| 102 | + } |
| 103 | + } |
| 104 | + |
| 105 | + // Video Processor Interface |
| 106 | + |
| 107 | + override fun onCapturerStarted(success: Boolean) { |
| 108 | + segmenterHelper = ImageSegmenterHelper(context = context, imageSegmenterListener = this) |
| 109 | + |
| 110 | + glThread = HandlerThread(GPU_THREAD).apply { start() } |
| 111 | + glHandler = Handler(glThread!!.looper) |
| 112 | + glHandler?.post { |
| 113 | + backgroundBlurGPUProcessor = BackgroundBlurGPUProcessor(context) |
| 114 | + backgroundBlurGPUProcessor?.init() |
| 115 | + } |
| 116 | + } |
| 117 | + |
| 118 | + override fun onCapturerStopped() { |
| 119 | + segmenterHelper?.destroyImageSegmenter() |
| 120 | + glHandler?.post { |
| 121 | + backgroundBlurGPUProcessor?.release() |
| 122 | + backgroundBlurGPUProcessor = null |
| 123 | + |
| 124 | + // Quit thread after cleanup |
| 125 | + glThread?.quitSafely() |
| 126 | + glThread = null |
| 127 | + glHandler = null |
| 128 | + } |
| 129 | + } |
| 130 | + |
| 131 | + override fun onFrameCaptured(videoFrame: VideoFrame) { |
| 132 | + val i420WebRTCBuffer = videoFrame.buffer.toI420() |
| 133 | + val width = videoFrame.buffer.width |
| 134 | + val height = videoFrame.buffer.height |
| 135 | + val rotation = FLOAT_ROTATION - videoFrame.rotation |
| 136 | + val videoFrameBuffer = i420WebRTCBuffer?.convertToABGR() |
| 137 | + |
| 138 | + i420WebRTCBuffer?.release() |
| 139 | + |
| 140 | + videoFrameBuffer?.let { |
| 141 | + rotationMap.synchronizedPut(videoFrame.timestampNs, rotation) |
| 142 | + frameBufferMap.synchronizedPut(videoFrame.timestampNs, it) |
| 143 | + segmenterHelper?.segmentFrame(it, width, height, videoFrame.timestampNs) |
| 144 | + } ?: { |
| 145 | + Log.e(TAG, "onFrameCaptured:: Video Frame was null!") |
| 146 | + sink?.onFrame(videoFrame) |
| 147 | + } |
| 148 | + } |
| 149 | + |
| 150 | + override fun setSink(sink: VideoSink?) { |
| 151 | + this.sink = sink |
| 152 | + } |
| 153 | + |
| 154 | + fun VideoFrame.I420Buffer.convertToABGR(): ByteBuffer { |
| 155 | + val dataYSize = dataY.limit() - dataY.position() |
| 156 | + val dataUSize = dataU.limit() - dataU.position() |
| 157 | + val dataVSize = dataV.limit() - dataV.position() |
| 158 | + |
| 159 | + val planeY = PlanePrimitive.create(strideY, dataY, dataYSize) |
| 160 | + val planeU = PlanePrimitive.create(strideU, dataU, dataUSize) |
| 161 | + val planeV = PlanePrimitive.create(strideV, dataV, dataVSize) |
| 162 | + |
| 163 | + val libYuvI420Buffer = I420Buffer.wrap(planeY, planeU, planeV, width, height) |
| 164 | + val libYuvABGRBuffer = AbgrBuffer.allocate(width, height) |
| 165 | + libYuvI420Buffer.convertTo(libYuvABGRBuffer) |
| 166 | + |
| 167 | + return libYuvABGRBuffer.asBuffer() |
| 168 | + } |
| 169 | + |
| 170 | + inline fun <reified K, V> LruCache<K, V>.synchronizedPut(key: K, value: V) { |
| 171 | + synchronized(this) { |
| 172 | + this.put(key, value) |
| 173 | + } |
| 174 | + } |
| 175 | + |
| 176 | + inline fun <reified K, V> LruCache<K, V>.synchronizedRemove(key: K) { |
| 177 | + synchronized(this) { |
| 178 | + this.remove(key) |
| 179 | + } |
| 180 | + } |
| 181 | + |
| 182 | + fun ByteArray.convertToWebRTCBuffer(width: Int, height: Int): JavaI420Buffer { |
| 183 | + val src = ByteBuffer.allocateDirect(this.size) |
| 184 | + src.put(this) |
| 185 | + |
| 186 | + val srcStride = width * INT_4 |
| 187 | + val yPlaneSize = width * height |
| 188 | + val uvPlaneSize = (width / 2) * (height / 2) |
| 189 | + |
| 190 | + val dstYStride = width |
| 191 | + val dstUStride = width / 2 |
| 192 | + val dstVStride = width / 2 |
| 193 | + |
| 194 | + val dstYBuffer = ByteBuffer.allocateDirect(yPlaneSize) |
| 195 | + val dstUBuffer = ByteBuffer.allocateDirect(uvPlaneSize) |
| 196 | + val dstVBuffer = ByteBuffer.allocateDirect(uvPlaneSize) |
| 197 | + |
| 198 | + YuvHelper.ABGRToI420( |
| 199 | + src, |
| 200 | + srcStride, |
| 201 | + dstYBuffer, |
| 202 | + dstYStride, |
| 203 | + dstUBuffer, |
| 204 | + dstUStride, |
| 205 | + dstVBuffer, |
| 206 | + dstVStride, |
| 207 | + width, |
| 208 | + height |
| 209 | + ) |
| 210 | + |
| 211 | + return JavaI420Buffer.wrap( |
| 212 | + width, |
| 213 | + height, |
| 214 | + dstYBuffer, |
| 215 | + dstYStride, |
| 216 | + dstUBuffer, |
| 217 | + dstUStride, |
| 218 | + dstVBuffer, |
| 219 | + dstVStride, |
| 220 | + null |
| 221 | + ) |
| 222 | + } |
| 223 | +} |
0 commit comments