refactor: inference pipeline

.gitignore

@@ -5,9 +5,11 @@
 *.swp
 .DS_Store
 .atom/
+.build/
 .buildlog/
 .history
 .svn/
+.swiftpm/
 migrate_working_dir/
 
 # IntelliJ related

ios/Podfile.lock

@@ -48,16 +48,28 @@ PODS:
     - SQLite.swift/standard (= 0.13.2)
   - SQLite.swift/standard (0.13.2)
   - Starscream (4.0.4)
-  - TensorFlowLiteC (2.14.0):
-    - TensorFlowLiteC/Core (= 2.14.0)
-  - TensorFlowLiteC/Core (2.14.0)
-  - TensorFlowLiteSwift (2.14.0):
-    - TensorFlowLiteSwift/Core (= 2.14.0)
-  - TensorFlowLiteSwift/Core (2.14.0):
-    - TensorFlowLiteC (= 2.14.0)
-  - tflite_flutter (0.1.0):
-    - Flutter
-    - TensorFlowLiteSwift
+  - TensorFlowLiteC (2.12.0):
+    - TensorFlowLiteC/Core (= 2.12.0)
+  - TensorFlowLiteC/Core (2.12.0)
+  - TensorFlowLiteC/CoreML (2.12.0):
+    - TensorFlowLiteC/Core
+  - TensorFlowLiteC/Metal (2.12.0):
+    - TensorFlowLiteC/Core
+  - TensorFlowLiteSwift (2.12.0):
+    - TensorFlowLiteSwift/Core (= 2.12.0)
+  - TensorFlowLiteSwift/Core (2.12.0):
+    - TensorFlowLiteC (= 2.12.0)
+  - TensorFlowLiteSwift/CoreML (2.12.0):
+    - TensorFlowLiteC/CoreML (= 2.12.0)
+    - TensorFlowLiteSwift/Core (= 2.12.0)
+  - TensorFlowLiteSwift/Metal (2.12.0):
+    - TensorFlowLiteC/Metal (= 2.12.0)
+    - TensorFlowLiteSwift/Core (= 2.12.0)
+  - tflite_flutter (0.0.1):
+    - Flutter
+    - TensorFlowLiteSwift (= 2.12.0)
+    - TensorFlowLiteSwift/CoreML (= 2.12.0)
+    - TensorFlowLiteSwift/Metal (= 2.12.0)
   - tflite_flutter_helper (0.0.1):
     - Flutter
 
@@ -133,11 +145,11 @@ SPEC CHECKSUMS:
   sensors_plus: 4b7c4bd9c9be2efbd575649368fe77ef09b429ad
   SQLite.swift: 4fc2be46c36392e3b87afe6fe7f1801c1daa07ef
   Starscream: 5178aed56b316f13fa3bc55694e583d35dd414d9
-  TensorFlowLiteC: a836de9b6e6018665ccc349538bdd2c3b80dd7a5
-  TensorFlowLiteSwift: f706b143b69d7bf433aba7b08049bdd11c321a10
-  tflite_flutter: bf9c7d5c7ef011034c607d943d432623673c1c11
+  TensorFlowLiteC: 20785a69299185a379ba9852b6625f00afd7984a
+  TensorFlowLiteSwift: 3a4928286e9e35bdd3e17970f48e53c80d25e793
+  tflite_flutter: 9433d086a3060431bbc9f3c7c20d017db0e72d08
   tflite_flutter_helper: 543b46b6bd064b21c92ea6e54bc0b29f1ce74cb5
 
-PODFILE CHECKSUM: dc4434bc7c18a619918d9cef0a52087a5f5985e1
+PODFILE CHECKSUM: c238ed21013585bf5a2c0c98c3005ba48f3d16e2
 
 COCOAPODS: 1.15.2

lib/core/services/repository.dart

@@ -21,7 +21,7 @@ abstract class Repository<I, O> {
   }
 
   Future<void> initialize();
-  Future<O?> runPredict(String id, I input, Map<String, dynamic>? params);
+  Future<O?> runPredict(Map<String, dynamic> params);
   Future<void> store(
       String id, I input, O output, Map<String, dynamic>? params);
   void reset() {

lib/core/utils/image_utils.dart

@@ -1,14 +1,15 @@
 import 'dart:isolate';
+import 'dart:typed_data';
 import 'package:camera/camera.dart';
 import 'package:image/image.dart' as image_lib;
-import 'isolate_utils.dart';
 
 // ignore: constant_identifier_names
 const IOS_BYTES_OFFSET = 28;
 
 /// ImageUtils
 class ImageUtils {
   /// Converts a [CameraImage] in YUV420 or BGRA8888 format to [Image] in RGB format
+  /// Ref: https://gist.github.com/Alby-o/fe87e35bc21d534c8220aed7df028e03?permalink_comment_id=4574528#gistcomment-4574528
   static image_lib.Image convertCameraImage(CameraImage cameraImage) {
     return switch (cameraImage.format.group) {
       ImageFormatGroup.yuv420 => convertYUV420ToImage(cameraImage),
@@ -23,14 +24,34 @@ class ImageUtils {
   static image_lib.Image convertBGRA8888ToImage(CameraImage cameraImage) {
     final plane = cameraImage.planes[0];
 
-    return image_lib.Image.fromBytes(
+    // Calculate the number of pixels
+    int numPixels = cameraImage.width * cameraImage.height;
+
+    // Create a Float32List for the RGB image without the alpha channel
+    Float32List floatList = Float32List(numPixels * 3); // 3 channels (RGB)
+
+    // Populate the floatList by ignoring the alpha channel
+    for (int i = 0, j = 0; i < plane.bytes.length; i += 4) {
+      floatList[j++] = plane.bytes[i] / 255.0; // Red
+      floatList[j++] = plane.bytes[i + 1] / 255.0; // Green
+      floatList[j++] = plane.bytes[i + 2] / 255.0; // Blue
+      // Skip alpha (i + 3)
+    }
+
+    // Determine the rowStride for the float32 image
+    int floatRowStride = plane.bytesPerRow * 3; // Adjust for float32 bytes
+
+    final img = image_lib.Image.fromBytes(
       width: cameraImage.width,
       height: cameraImage.height,
-      bytes: plane.bytes.buffer,
-      rowStride: plane.bytesPerRow,
-      bytesOffset: IOS_BYTES_OFFSET,
-      order: image_lib.ChannelOrder.bgra,
+      bytes: floatList.buffer,
+      rowStride: floatRowStride,
+      // bytesOffset: IOS_BYTES_OFFSET,
+      order: image_lib.ChannelOrder.rgb,
+      format: image_lib.Format.float32,
     );
+    return image_lib.copyResize(img,
+        width: 256, height: 256, interpolation: image_lib.Interpolation.linear);
   }
 
   /// Converts a [CameraImage] in YUV420 format to [image_lib.Image] in RGB format
@@ -106,22 +127,20 @@ class ImageUtils {
   // }
 
   /// Convert a [Image] to PNG
-  static Future<List<int>> convert(
-      {required IsolateUtils isolateUtils, required CameraImage image}) async {
-    final responsePort = ReceivePort();
-
-    toPng(Map<String, dynamic> params) =>
-        image_lib.encodePng(ImageUtils.convertCameraImage(params['image']));
-
-    isolateUtils.sendMessage(
-      handler: toPng,
-      params: {
-        'image': image,
-      },
-      sendPort: isolateUtils.sendPort,
-      responsePort: responsePort,
-    );
-
-    return await responsePort.first;
+  static Future<List<int>> convert({required CameraImage image}) async {
+    //final responsePort = ReceivePort();
+
+    final encodedImage = Isolate.run(
+        () => image_lib.encodePng(ImageUtils.convertCameraImage(image)));
+    // isolateUtils.sendMessage(
+    //   handler: toPng,
+    //   params: {
+    //     'image': image,
+    //   },
+    //   sendPort: isolateUtils.sendPort,
+    //   responsePort: responsePort,
+    // );
+
+    return await encodedImage;
   }
 }

lib/features/assessment/config/dev/dev.dart

@@ -27,7 +27,7 @@ class DEV implements BaseAssessment {
   int get trials => 1;
 
   @override
-  int get trialDuration => 5;
+  int get trialDuration => 500;
 
   @override
   List<String> get conditionNames => [

lib/features/inference/bloc/inference_bloc.dart

@@ -62,8 +62,13 @@ class InferenceBloc<I extends Identifiable, O extends AiModelOutput>
     }
     if (processing) return null;
     processing = true;
-    final data = await inferenceRepository.runPredict(
-        event.id, event.input, event.params);
+    final params = {
+      "id": event.id,
+      "input": event.input,
+      "params": event.params,
+    };
+
+    final data = await inferenceRepository.runPredict(params);
     processing = false;
     if (isReady && !isClosed && !emit.isDone) {
       add(InferenceOutput(
@@ -121,6 +126,6 @@ class InferenceBloc<I extends Identifiable, O extends AiModelOutput>
   void onError(Object error, StackTrace stackTrace) {
     // Always call super.onError with the current error
     super.onError(error, stackTrace);
-    log('InferenceBloc Error: $stackTrace $error');
+    log('InferenceBloc Error: $error $stackTrace');
   }
 }

lib/features/inference/bloc/inference_event.dart

@@ -17,7 +17,7 @@ class InferenceInit<I, O> extends InferenceEvent<I, O> {
 }
 
 class InferenceStart<I, O> extends InferenceEvent<I, O> {
-  final Map<String, dynamic>? params;
+  final Map<String, dynamic> params;
   const InferenceStart(super.id, this.params);
 
   @override
@@ -26,11 +26,11 @@ class InferenceStart<I, O> extends InferenceEvent<I, O> {
 
 class InferenceInput<I, O> extends InferenceEvent<I, O> {
   final I input;
-  final Map<String, dynamic>? params;
+  final Map<String, dynamic> params;
   const InferenceInput(
     super.id, {
     required this.input,
-    this.params,
+    required this.params,
   });
 
   @override

lib/features/inference/constants/model_file.dart

@@ -1,10 +1,11 @@
 mixin ModelFile {
-  static const String faceDetection = 'models/face_detection.tflite';
-  static const String faceMesh = 'models/face_landmark.tflite';
-  static const String hands = 'models/hand_landmark.tflite';
-  static const String poseDetection = 'models/pose_landmark_heavy.tflite';
+  static const String faceDetection = 'assets/models/face_detection.tflite';
+  static const String faceMesh = 'assets/models/face_landmark.tflite';
+  static const String hands = 'assets/models/hand_landmark.tflite';
+  static const String poseDetection =
+      'assets/models/pose_landmark_heavy.tflite';
   static const String poseClassification =
-      'models/pose_classification.tflite'; //TODO:
+      'assets/models/pose_classification.tflite'; //TODO:
 }
 
 enum InferenceType {

lib/features/inference/service/ai_model.dart

@@ -4,12 +4,12 @@ import 'package:visualpt/features/inference/bloc/inference_bloc.dart';
 
 // ignore: must_be_immutable
 abstract class AiModel<I, O extends AiModelOutput> extends Equatable {
-  AiModel({this.interpreter});
+  AiModel();
 
   final inputShapes = <List<int>>[];
-  final inputTypes = <TfLiteType>[];
+  final inputTypes = <TensorType>[];
   final outputShapes = <List<int>>[];
-  final outputTypes = <TfLiteType>[];
+  final outputTypes = <TensorType>[];
 
   Interpreter? interpreter;
 
@@ -19,7 +19,7 @@ abstract class AiModel<I, O extends AiModelOutput> extends Equatable {
   int get getAddress;
 
   Future<void> loadModel();
-  O? predict(String id, I input, Map<String, dynamic>? params);
+  O? predict(Map<String, dynamic> params);
 }
 
 abstract class AiModelOutput extends Identifiable {

lib/features/inference/service/face_detection/face_detection_service.dart

@@ -7,7 +7,6 @@ import 'package:tflite_flutter/tflite_flutter.dart';
 import 'package:tflite_flutter_helper/tflite_flutter_helper.dart';
 
 import '../../constants/model_file.dart';
-import '../../../../core/utils/image_utils.dart';
 import '../ai_model.dart';
 import 'anchors.dart';
 import 'generate_anchors.dart';
@@ -85,13 +84,15 @@ class FaceDetection extends AiModel<image_lib.Image, FaceBboxOutput> {
   }
 
   @override
-  FaceBboxOutput? predict(
-      String id, image_lib.Image input, Map<String, dynamic>? params) {
+  FaceBboxOutput? predict(Map<String, dynamic> params) {
     if (interpreter == null) {
       log('Interpreter not initialized');
       return null;
     }
 
+    final String id = params['id'];
+    image_lib.Image input = params['input'];
+
     final options = OptionsFace(
         numClasses: 1,
         numBoxes: 896,
@@ -113,7 +114,7 @@ class FaceDetection extends AiModel<image_lib.Image, FaceBboxOutput> {
       input = image_lib.copyRotate(input, angle: -90);
       input = image_lib.flipHorizontal(input);
     }
-    final tensorImage = TensorImage(TfLiteType.float32);
+    final tensorImage = TensorImage(TensorType.float32);
     tensorImage.loadImage(input);
     final inputImage = getProcessedImage(tensorImage);
 
@@ -167,16 +168,6 @@ class FaceDetection extends AiModel<image_lib.Image, FaceBboxOutput> {
   }
 }
 
-FaceBboxOutput? runFaceDetector(Map<String, dynamic> params) {
-  final faceDetection = FaceDetection(
-      interpreter: Interpreter.fromAddress(params['detectorAddress']));
-  final id = params['id'] as String;
-  final input = ImageUtils.convertCameraImage(params['input']);
-  final result = faceDetection.predict(id, input, {});
-
-  return result;
-}
-
 class FaceBboxOutput extends AiModelOutput {
   const FaceBboxOutput(super.id, super.output);
 

lib/features/inference/service/face_mesh/face_mesh_service.dart

@@ -7,7 +7,6 @@ import 'package:tflite_flutter/tflite_flutter.dart';
 import 'package:tflite_flutter_helper/tflite_flutter_helper.dart';
 
 import '../../constants/model_file.dart';
-import '../../../../core/utils/image_utils.dart';
 import '../ai_model.dart';
 
 // ignore: must_be_immutable
@@ -57,18 +56,19 @@ class FaceMesh extends AiModel<image_lib.Image, FaceOutput> {
   }
 
   @override
-  FaceOutput? predict(
-      String id, image_lib.Image input, Map<String, dynamic>? params) {
+  FaceOutput? predict(Map<String, dynamic> params) {
     if (interpreter == null) {
       log('Interpreter not initialized');
       return null;
     }
+    final String id = params['id'];
+    image_lib.Image input = params['input'];
 
     if (Platform.isAndroid) {
       input = image_lib.copyRotate(input, angle: -90);
       input = image_lib.flipHorizontal(input);
     }
-    final tensorImage = TensorImage(TfLiteType.float32);
+    final tensorImage = TensorImage(TensorType.float32);
     tensorImage.loadImage(input);
     final inputImage = getProcessedImage(tensorImage);
 
@@ -100,15 +100,6 @@ class FaceMesh extends AiModel<image_lib.Image, FaceOutput> {
   }
 }
 
-FaceOutput? runFaceMesh(Map<String, dynamic> params) {
-  final faceMesh =
-      FaceMesh(interpreter: Interpreter.fromAddress(params['detectorAddress']));
-  final image = ImageUtils.convertCameraImage(params['input']);
-  final result = faceMesh.predict(params["id"], image, {});
-
-  return result;
-}
-
 class FaceOutput extends AiModelOutput {
   const FaceOutput(super.id, super.output);