diff --git a/REUSE.toml b/REUSE.toml index 43ee0f4b84a..26046715c38 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -34,9 +34,3 @@ path = "app/src/main/res/raw/next_voice_message_doodle.ogg" precedence = "aggregate" SPDX-FileCopyrightText = "2024 Paciosoft" SPDX-License-Identifier = "CC-BY-4.0" - -[[annotations]] -path = "app/src/main/assets/selfie_segmenter.tflite" -precedence = "aggregate" -SPDX-FileCopyrightText = "2025 Google" -SPDX-License-Identifier = "Apache-2.0" \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index 131527016e6..ece4caa759d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -16,7 +16,6 @@ plugins { id "org.jetbrains.kotlin.plugin.compose" version "2.3.0" id "org.jetbrains.kotlin.kapt" id 'com.google.devtools.ksp' version '2.3.4' - id "de.undercouch.download" version "5.6.0" // Download task and extension for retrieving files during a build } apply plugin: 'com.android.application' @@ -369,10 +368,6 @@ dependencies { testImplementation("com.squareup.okhttp3:mockwebserver:$okhttpVersion") testImplementation("com.google.dagger:hilt-android-testing:2.57.2") testImplementation("org.robolectric:robolectric:4.16") - - // Computer Vision - for background effects during video calls - implementation 'com.google.mediapipe:tasks-vision:0.10.26' - implementation 'org.opencv:opencv:4.12.0' } tasks.register('installGitHooks', Copy) { diff --git a/app/src/main/assets/selfie_segmenter.tflite b/app/src/main/assets/selfie_segmenter.tflite deleted file mode 100644 index a4ebd4777ed..00000000000 Binary files a/app/src/main/assets/selfie_segmenter.tflite and /dev/null differ diff --git a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt index 45869d73ed8..095178fefbf 100644 --- a/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt +++ b/app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt @@ -74,9 +74,6 @@ import com.nextcloud.talk.call.ReactionAnimator import com.nextcloud.talk.call.components.ParticipantGrid import com.nextcloud.talk.call.components.SelfVideoView import com.nextcloud.talk.call.components.screenshare.ScreenShareComponent -import com.nextcloud.talk.camera.BackgroundBlurFrameProcessor -import com.nextcloud.talk.camera.BlurBackgroundViewModel -import com.nextcloud.talk.camera.BlurBackgroundViewModel.BackgroundBlurOn import com.nextcloud.talk.chat.ChatActivity import com.nextcloud.talk.data.user.model.User import com.nextcloud.talk.databinding.CallActivityBinding @@ -219,7 +216,6 @@ class CallActivity : CallBaseActivity() { var audioManager: WebRtcAudioManager? = null var callRecordingViewModel: CallRecordingViewModel? = null var raiseHandViewModel: RaiseHandViewModel? = null - val blurBackgroundViewModel: BlurBackgroundViewModel = BlurBackgroundViewModel() private var mReceiver: BroadcastReceiver? = null private var peerConnectionFactory: PeerConnectionFactory? = null private var screenSharePeerConnectionFactory: PeerConnectionFactory? = null @@ -446,7 +442,6 @@ class CallActivity : CallBaseActivity() { initRaiseHandViewModel() initCallRecordingViewModel(intent.extras!!.getInt(KEY_RECORDING_STATE)) - initBackgroundBlurViewModel() initClickListeners(isModerator, isOneToOneConversation) binding!!.microphoneButton.setOnTouchListener(MicrophoneButtonTouchListener()) @@ -539,26 +534,6 @@ class CallActivity : CallBaseActivity() { } } - private fun initBackgroundBlurViewModel() { - blurBackgroundViewModel.viewState.observe(this) { state -> - val frontFacing = isCameraFrontFacing(cameraEnumerator) - if (frontFacing == null) { - Log.e(TAG, "Camera not found") - return@observe - } - - val isOn = state == BackgroundBlurOn - - val processor = if (isOn) { - BackgroundBlurFrameProcessor(context) - } else { - null - } - - videoSource?.setVideoProcessor(processor) - } - } - private fun processExtras(extras: Bundle) { roomId = extras.getString(KEY_ROOM_ID, "") roomToken = extras.getString(KEY_ROOM_TOKEN, "") @@ -1214,30 +1189,6 @@ class CallActivity : CallBaseActivity() { return null } - private fun isCameraFrontFacing(enumerator: CameraEnumerator?): Boolean? { - if (enumerator == null) { - return false - } - - val deviceNames = enumerator.deviceNames - - // First, try to find front facing camera - for (deviceName in deviceNames) { - if (enumerator.isFrontFacing(deviceName)) { - return true - } - } - - // Front facing camera not found, try something else - for (deviceName in deviceNames) { - if (!enumerator.isFrontFacing(deviceName)) { - return false - } - } - - return null - } - fun onMicrophoneClick() { if (!canPublishAudioStream) { microphoneOn = false @@ -1319,7 +1270,6 @@ class CallActivity : CallBaseActivity() { binding!!.cameraButton.setImageResource(R.drawable.ic_videocam_white_24px) } else { binding!!.cameraButton.setImageResource(R.drawable.ic_videocam_off_white_24px) - blurBackgroundViewModel.turnOffBlur() } toggleMedia(videoOn, true) } else if (shouldShowRequestPermissionRationale(Manifest.permission.CAMERA)) { @@ -1396,10 +1346,6 @@ class CallActivity : CallBaseActivity() { raiseHandViewModel!!.clickHandButton() } - fun toggleBackgroundBlur() { - blurBackgroundViewModel.toggleBackgroundBlur() - } - public override fun onDestroy() { if (signalingMessageReceiver != null) { signalingMessageReceiver!!.removeListener(localParticipantMessageListener) diff --git a/app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt b/app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt deleted file mode 100644 index 1c25249b00b..00000000000 --- a/app/src/main/java/com/nextcloud/talk/camera/BackgroundBlurFrameProcessor.kt +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Nextcloud Talk - Android Client - * - * SPDX-FileCopyrightText: 2025 Julius Linus - * SPDX-License-Identifier: GPL-3.0-or-later - */ - -package com.nextcloud.talk.camera - -import android.content.Context -import android.graphics.Bitmap -import android.util.Log -import androidx.core.graphics.createBitmap -import org.opencv.android.OpenCVLoader -import org.opencv.android.Utils -import org.opencv.core.CvType -import org.opencv.core.Mat -import org.opencv.core.Point -import org.opencv.core.Size -import org.opencv.imgproc.Imgproc -import org.webrtc.JavaI420Buffer -import org.webrtc.VideoFrame -import org.webrtc.VideoProcessor -import org.webrtc.VideoSink -import org.webrtc.YuvHelper -import java.nio.ByteBuffer - -class BackgroundBlurFrameProcessor(val context: Context) : - VideoProcessor, - ImageSegmenterHelper.SegmenterListener { - - companion object { - val TAG: String = this::class.java.simpleName - const val ROT_360 = 360 - const val KERNEL_SIZE = 25.0 // must be odd - const val NV21_HEIGHT_MULTI = 1.5 - } - - init { - if (OpenCVLoader.initLocal()) { - Log.i("OpenCV", "OpenCV library loaded successfully.") - } else { - Log.e("OpenCV", "OpenCV library not found.") - } - } - - private var sink: VideoSink? = null - private var segmenterHelper: ImageSegmenterHelper? = null - private val frameQueue = ArrayDeque() - - // SegmentationListener Interface - - override fun onError(error: String, errorCode: Int) { - Log.e(TAG, "Error $errorCode: $error") - } - - override fun onResults(resultBundle: ImageSegmenterHelper.ResultBundle) { - if (frameQueue.isEmpty()) { - return - } - - val maskMat = resultBundle.mask - - val frameBitmap: Bitmap = frameQueue.removeFirst() - - val frameMat = Mat() - val blurredMat = Mat() - - try { - Utils.bitmapToMat(frameBitmap, frameMat) - - val blurredMat = frameMat.clone() - Imgproc.GaussianBlur( - blurredMat, - blurredMat, - Size(KERNEL_SIZE, KERNEL_SIZE), - 0.0, - 0.0 - ) - - Imgproc.resize( - maskMat, - maskMat, - frameMat.size(), - .0, - .0, - Imgproc.INTER_LINEAR - ) - - // Copies pixels from `frameMat` to `blurredMat` ONLY where `grayMaskMat` is non-zero. - // This keeps the background blurred, while leaving the foreground clear - frameMat.copyTo(blurredMat, maskMat) - - val finalFrame = blurredMat.toVideoFrame(resultBundle.inferenceTime) - - if (finalFrame == null) { - Log.e(TAG, "Frame was null") - } - - sink?.onFrame(finalFrame) - - finalFrame?.release() - } finally { - frameMat.release() - blurredMat.release() - maskMat.release() - } - } - - // Video Processor Interface - - override fun onCapturerStarted(success: Boolean) { - segmenterHelper = ImageSegmenterHelper(context = context, imageSegmenterListener = this) - } - - override fun onCapturerStopped() { - segmenterHelper?.destroyImageSegmenter() - } - - override fun onFrameCaptured(videoFrame: VideoFrame) { - val buffer = videoFrame.buffer.toI420() - - if (buffer == null) { - videoFrame.release() - return - } - - val frameMatrix = buffer.toMat() - if (frameMatrix == null) { - videoFrame.release() - return - } - - val center = Point(frameMatrix.width() / 2.0, frameMatrix.height() / 2.0) - - var rotationMat = Mat() - - try { - // weirdly rotation is 270 degree in portrait and 180 degree in landscape, no idea why - // regardless if this behavior is device dependant, this calculation should correct the orientation - val angle = ROT_360 - videoFrame.rotation.toDouble() - rotationMat = Imgproc.getRotationMatrix2D( - center, - angle, - 1.0 - ) - - Imgproc.warpAffine(frameMatrix, frameMatrix, rotationMat, frameMatrix.size()) - - val preProcessedBitmap = createBitmap( - frameMatrix.width(), - frameMatrix.height(), - Bitmap.Config.ARGB_8888 - ) - - Utils.matToBitmap(frameMatrix, preProcessedBitmap) - - if (frameQueue.isEmpty()) { - frameQueue.add(preProcessedBitmap) - segmenterHelper?.segmentLiveStreamFrame(preProcessedBitmap, videoFrame.timestampNs) - } else { - // A frame is already being processed, drop this one - preProcessedBitmap.recycle() - } - } finally { - frameMatrix.release() - rotationMat.release() - } - } - - override fun setSink(sink: VideoSink?) { - this.sink = sink - } - - private fun Mat.toVideoFrame(time: Long): VideoFrame? = - runCatching { - val i420Mat = Mat() - Imgproc.cvtColor(this, i420Mat, Imgproc.COLOR_RGBA2YUV_I420) - - // Get the raw bytes from the new I420 Mat to i420ByteArray - val i420ByteArray = ByteArray((i420Mat.total() * i420Mat.elemSize()).toInt()) - i420Mat.get(0, 0, i420ByteArray) - - val width = this.width() - val height = this.height() - - val yPlaneSize = width * height - val uvPlaneSize = (width / 2) * (height / 2) - - val yBuffer = ByteBuffer.allocateDirect(yPlaneSize) - val uBuffer = ByteBuffer.allocateDirect(uvPlaneSize) - val vBuffer = ByteBuffer.allocateDirect(uvPlaneSize) - - yBuffer.put(i420ByteArray, 0, yPlaneSize) - uBuffer.put(i420ByteArray, yPlaneSize, uvPlaneSize) - vBuffer.put(i420ByteArray, yPlaneSize + uvPlaneSize, uvPlaneSize) - - yBuffer.rewind() - uBuffer.rewind() - vBuffer.rewind() - - // Create the I420Buffer using the separate planes - val finalFrameBuffer = JavaI420Buffer.wrap( - width, - height, - yBuffer, - width, - uBuffer, - width / 2, - vBuffer, - width / 2, - null - ) - - i420Mat.release() - - return VideoFrame(finalFrameBuffer, 0, time) - }.getOrElse { throwable -> - Log.e(TAG, "Error in Mat.toVideoFrame $throwable") - - null - } - - private fun VideoFrame.I420Buffer.toMat(): Mat? = - runCatching { - val chromaWidth = (width + 1) / 2 - val chromaHeight = (height + 1) / 2 - val minSize = width * height + chromaWidth * chromaHeight * 2 - - val nv12ByteBuffer = ByteBuffer.allocateDirect(minSize) - YuvHelper.I420ToNV12( - this.dataY, - this.strideY, - this.dataU, - this.strideU, - this.dataV, - this.strideV, - nv12ByteBuffer, - width, - height - ) - - val mat = Mat( - (height * NV21_HEIGHT_MULTI).toInt(), - width, - CvType.CV_8UC1 // 8 bit unsigned 1 channel - ) - - mat.put(0, 0, nv12ByteBuffer.array()) - - Imgproc.cvtColor(mat, mat, Imgproc.COLOR_YUV2RGBA_NV12) - - this.release() - - mat - }.getOrElse { throwable -> - Log.e(TAG, "Error in VideoFrame.I420Buffer.toMat $throwable") - - null - } -} diff --git a/app/src/main/java/com/nextcloud/talk/camera/BlurBackgroundViewModel.kt b/app/src/main/java/com/nextcloud/talk/camera/BlurBackgroundViewModel.kt deleted file mode 100644 index 0d05225d04e..00000000000 --- a/app/src/main/java/com/nextcloud/talk/camera/BlurBackgroundViewModel.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Nextcloud Talk - Android Client - * - * SPDX-FileCopyrightText: 2025 Julius Linus - * SPDX-License-Identifier: GPL-3.0-or-later - */ - -package com.nextcloud.talk.camera - -import androidx.lifecycle.LiveData -import androidx.lifecycle.MutableLiveData -import androidx.lifecycle.ViewModel - -class BlurBackgroundViewModel : ViewModel() { - - sealed interface ViewState - - object BackgroundBlurOn : ViewState - object BackgroundBlurOff : ViewState - - private val _viewState: MutableLiveData = MutableLiveData(BackgroundBlurOff) - val viewState: LiveData - get() = _viewState - - fun toggleBackgroundBlur() { - val isOn = _viewState.value == BackgroundBlurOn - - if (isOn) { - _viewState.value = BackgroundBlurOff - } else { - _viewState.value = BackgroundBlurOn - } - } - - fun turnOffBlur() { - _viewState.value = BackgroundBlurOff - } -} diff --git a/app/src/main/java/com/nextcloud/talk/camera/ImageSegmenterHelper.kt b/app/src/main/java/com/nextcloud/talk/camera/ImageSegmenterHelper.kt deleted file mode 100644 index 584e78c2f60..00000000000 --- a/app/src/main/java/com/nextcloud/talk/camera/ImageSegmenterHelper.kt +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Nextcloud Talk - Android Client - * - * SPDX-FileCopyrightText: 2025 Julius Linus - * SPDX-License-Identifier: GPL-3.0-or-later - */ - -package com.nextcloud.talk.camera - -import android.content.Context -import android.graphics.Bitmap -import android.util.Log -import com.google.mediapipe.framework.image.BitmapImageBuilder -import com.google.mediapipe.framework.image.ByteBufferExtractor -import com.google.mediapipe.framework.image.MPImage -import com.google.mediapipe.tasks.core.BaseOptions -import com.google.mediapipe.tasks.core.Delegate -import com.google.mediapipe.tasks.vision.core.RunningMode -import com.google.mediapipe.tasks.vision.imagesegmenter.ImageSegmenter -import com.google.mediapipe.tasks.vision.imagesegmenter.ImageSegmenterResult -import org.opencv.core.Core -import org.opencv.core.CvType -import org.opencv.core.Mat -import org.opencv.core.Scalar -import java.nio.ByteBuffer - -class ImageSegmenterHelper(val context: Context, var imageSegmenterListener: SegmenterListener? = null) { - - private var imageSegmenter: ImageSegmenter? = null - - init { - setupImageSegmenter() - } - - // Segmenter must be closed when creating a new one to avoid returning results to a non-existent object - fun destroyImageSegmenter() { - imageSegmenter?.close() - imageSegmenter = null - } - - /** - * Initialize the image segmenter using current settings on the - * thread that is using it. CPU can be used with detectors - * that are created on the main thread and used on a background thread, but - * the GPU delegate needs to be used on the thread that initialized the - * segmenter - * - * @throws IllegalStateException - */ - fun setupImageSegmenter() { - val baseOptionsBuilder = BaseOptions.builder().apply { - setDelegate(Delegate.CPU) - setModelAssetPath(MODEL_SELFIE_SEGMENTER_PATH) - } - - if (imageSegmenterListener == null) { - throw IllegalStateException("ImageSegmenterListener must be set.") - } - - runCatching { - val baseOptions = baseOptionsBuilder.build() - val optionsBuilder = ImageSegmenter.ImageSegmenterOptions.builder() - .setRunningMode(RunningMode.LIVE_STREAM) - .setBaseOptions(baseOptions) - .setOutputCategoryMask(true) - .setOutputConfidenceMasks(false) - .setResultListener(this::returnSegmentationResult) - .setErrorListener(this::returnSegmentationHelperError) - - imageSegmenter = ImageSegmenter.createFromOptions(context, optionsBuilder.build()) - }.getOrElse { e -> - when (e) { - is IllegalStateException -> { - imageSegmenterListener?.onError( - "Image segmenter failed to initialize. See error logs for details" - ) - Log.e(TAG, "Image segmenter failed to load model with error: ${e.message}") - } - - is RuntimeException -> { - // This occurs if the model being used does not support GPU - imageSegmenterListener?.onError( - "Image segmenter failed to initialize. See error logs for details", - GPU_ERROR - ) - Log.e(TAG, "Image segmenter failed to load model with error: ${e.message}") - } - } - } - } - - /** - * Runs image segmentation on live streaming cameras frame-by-frame and - * returns the results asynchronously to the given `imageSegmenterListener` - * - * @throws IllegalArgumentException - */ - fun segmentLiveStreamFrame(bitmap: Bitmap, videoFrameTimeStamp: Long) { - val mpImage = BitmapImageBuilder(bitmap).build() - - imageSegmenter?.segmentAsync(mpImage, videoFrameTimeStamp) - } - - // MPImage isn't necessary, but the listener requires it - private fun returnSegmentationResult(result: ImageSegmenterResult, image: MPImage) { - // We only need the first mask for this sample because we are using - // the OutputType CATEGORY_MASK, which only provides a single mask. - val mpImage = result.categoryMask().get() - - val mask = ByteBufferExtractor.extract(mpImage) - - val data = ByteArray(mask.capacity()) - - (mask.duplicate().rewind() as ByteBuffer).get(data) - - val mat = Mat( - mpImage.height, - mpImage.width, - CvType.CV_8UC1 // 8 bit unsigned 1 channel - ) - - mat.put(0, 0, data) - - Core.bitwise_not(mat, mat) - - Core.multiply(mat, Scalar(RGB_MAX), mat) - - imageSegmenterListener?.onResults( - ResultBundle( - mat, - result.timestampMs() // videoFrameTimeStamp - ) - ) - } - - // Return errors thrown during segmentation to this ImageSegmenterHelper's caller - private fun returnSegmentationHelperError(error: RuntimeException) { - imageSegmenterListener?.onError( - error.message ?: "An unknown error has occurred" - ) - } - - // Wraps results from inference, the time it takes for inference to be performed. - data class ResultBundle(val mask: Mat, val inferenceTime: Long) - - companion object { - const val OTHER_ERROR = 0 - const val GPU_ERROR = 1 - const val MODEL_SELFIE_SEGMENTER_PATH = "selfie_segmenter.tflite" - const val RGB_MAX = 255.0 - private const val TAG = "ImageSegmenterHelper" - } - - interface SegmenterListener { - fun onError(error: String, errorCode: Int = OTHER_ERROR) - fun onResults(resultBundle: ResultBundle) - } -} diff --git a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt index eab8a7d06f2..74e1d9306c0 100644 --- a/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt +++ b/app/src/main/java/com/nextcloud/talk/ui/dialog/MoreCallActionsDialog.kt @@ -18,7 +18,6 @@ import com.google.android.material.bottomsheet.BottomSheetDialog import com.nextcloud.talk.R import com.nextcloud.talk.activities.CallActivity import com.nextcloud.talk.application.NextcloudTalkApplication -import com.nextcloud.talk.camera.BlurBackgroundViewModel import com.nextcloud.talk.databinding.DialogMoreCallActionsBinding import com.nextcloud.talk.raisehand.viewmodel.RaiseHandViewModel import com.nextcloud.talk.ui.theme.ViewThemeUtils @@ -77,12 +76,6 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee } else { binding.raiseHand.visibility = View.GONE } - - binding.backgroundBlur.visibility = if (callActivity.isVoiceOnlyCall) { - View.GONE - } else { - View.VISIBLE - } } private fun initClickListeners() { @@ -93,10 +86,6 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee binding.raiseHand.setOnClickListener { callActivity.clickRaiseOrLowerHandButton() } - - binding.backgroundBlur.setOnClickListener { - callActivity.toggleBackgroundBlur() - } } private fun initEmojiBar() { @@ -192,18 +181,6 @@ class MoreCallActionsDialog(private val callActivity: CallActivity) : BottomShee else -> {} } } - - callActivity.blurBackgroundViewModel.viewState.observe(this) { state -> - when (state) { - BlurBackgroundViewModel.BackgroundBlurOff -> { - binding.backgroundBlurText.text = context.getText(R.string.turn_on_background_blur) - } - - BlurBackgroundViewModel.BackgroundBlurOn -> { - binding.backgroundBlurText.text = context.getText(R.string.turn_off_background_blur) - } - } - } } companion object { diff --git a/app/src/main/res/drawable/background_replace_24px.xml b/app/src/main/res/drawable/background_replace_24px.xml deleted file mode 100644 index 50b578e026a..00000000000 --- a/app/src/main/res/drawable/background_replace_24px.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - diff --git a/app/src/main/res/layout/dialog_more_call_actions.xml b/app/src/main/res/layout/dialog_more_call_actions.xml index 0490bacac6b..74df6cf9857 100644 --- a/app/src/main/res/layout/dialog_more_call_actions.xml +++ b/app/src/main/res/layout/dialog_more_call_actions.xml @@ -116,37 +116,4 @@ - - - - - - - - diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 5c15d936afc..bc5ad1f74c1 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -904,7 +904,5 @@ How to translate with transifex: Please enable it in the app settings Location services disabled Please enable location services (GPS) to use this feature - Turn on background blur - Turn off background blur Please continue login in browser