From 92cbd1607a12611dd1a49fecfa93d71ea55993a2 Mon Sep 17 00:00:00 2001 From: charlieforward9 Date: Fri, 11 Oct 2024 21:42:56 -0400 Subject: [PATCH 1/2] refactor: bump tflite_flutter --- lib/features/inference/service/ai_model.dart | 4 +- .../face_detection_service.dart | 2 +- .../service/face_mesh/face_mesh_service.dart | 2 +- .../service/hands/hands_service.dart | 2 +- .../service/video_detection_repository.dart | 2 +- pubspec.lock | 55 +++++++++++-------- pubspec.yaml | 8 +-- 7 files changed, 41 insertions(+), 34 deletions(-) diff --git a/lib/features/inference/service/ai_model.dart b/lib/features/inference/service/ai_model.dart index ceae761c..599c9ccb 100644 --- a/lib/features/inference/service/ai_model.dart +++ b/lib/features/inference/service/ai_model.dart @@ -7,9 +7,9 @@ abstract class AiModel extends Equatable { AiModel({this.interpreter}); final inputShapes = >[]; - final inputTypes = []; + final inputTypes = []; final outputShapes = >[]; - final outputTypes = []; + final outputTypes = []; Interpreter? interpreter; diff --git a/lib/features/inference/service/face_detection/face_detection_service.dart b/lib/features/inference/service/face_detection/face_detection_service.dart index 291c899c..8622b3b4 100644 --- a/lib/features/inference/service/face_detection/face_detection_service.dart +++ b/lib/features/inference/service/face_detection/face_detection_service.dart @@ -113,7 +113,7 @@ class FaceDetection extends AiModel { input = image_lib.copyRotate(input, angle: -90); input = image_lib.flipHorizontal(input); } - final tensorImage = TensorImage(TfLiteType.float32); + final tensorImage = TensorImage(TensorType.float32); tensorImage.loadImage(input); final inputImage = getProcessedImage(tensorImage); diff --git a/lib/features/inference/service/face_mesh/face_mesh_service.dart b/lib/features/inference/service/face_mesh/face_mesh_service.dart index cff59cdc..571d839c 100644 --- a/lib/features/inference/service/face_mesh/face_mesh_service.dart +++ b/lib/features/inference/service/face_mesh/face_mesh_service.dart @@ -68,7 +68,7 @@ class FaceMesh extends AiModel { input = image_lib.copyRotate(input, angle: -90); input = image_lib.flipHorizontal(input); } - final tensorImage = TensorImage(TfLiteType.float32); + final tensorImage = TensorImage(TensorType.float32); tensorImage.loadImage(input); final inputImage = getProcessedImage(tensorImage); diff --git a/lib/features/inference/service/hands/hands_service.dart b/lib/features/inference/service/hands/hands_service.dart index 22c571e1..49ae1ea9 100644 --- a/lib/features/inference/service/hands/hands_service.dart +++ b/lib/features/inference/service/hands/hands_service.dart @@ -70,7 +70,7 @@ class Hands extends AiModel { input = image_lib.copyRotate(input, angle: -90); input = image_lib.flipHorizontal(input); } - final tensorImage = TensorImage(TfLiteType.float32); + final tensorImage = TensorImage(TensorType.float32); tensorImage.loadImage(input); final inputImage = getProcessedImage(tensorImage); diff --git a/lib/features/video/service/video_detection_repository.dart b/lib/features/video/service/video_detection_repository.dart index 1ebfbc70..75451e1b 100644 --- a/lib/features/video/service/video_detection_repository.dart +++ b/lib/features/video/service/video_detection_repository.dart @@ -121,7 +121,7 @@ class VideoDetectionRepository TensorImage preProcessData(CameraImage input) { final image = ImageUtils.convertCameraImage(input); - var tensorImage = TensorImage(TfLiteType.float32); + var tensorImage = TensorImage(TensorType.float32); tensorImage.loadImage(image); final imageProcessor = ImageProcessorBuilder() .add(ResizeOp(256, 256, ResizeMethod.BILINEAR)) diff --git a/pubspec.lock b/pubspec.lock index 8588a93f..04ca9c0c 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -261,18 +261,18 @@ packages: dependency: "direct main" description: name: camera - sha256: dfa8fc5a1adaeb95e7a54d86a5bd56f4bb0e035515354c8ac6d262e35cec2ec8 + sha256: "26ff41045772153f222ffffecba711a206f670f5834d40ebf5eed3811692f167" url: "https://pub.dev" source: hosted - version: "0.10.6" - camera_android: + version: "0.11.0+2" + camera_android_camerax: dependency: transitive description: - name: camera_android - sha256: "7b0aba6398afa8475e2bc9115d976efb49cf8db781e922572d443795c04a4f4f" + name: camera_android_camerax + sha256: e3627fdc2132d89212b8a8676679f5b07008c7e3d8ae00cea775c3397f9e742b url: "https://pub.dev" source: hosted - version: "0.10.9+1" + version: "0.6.10" camera_avfoundation: dependency: transitive description: @@ -326,9 +326,11 @@ packages: description: name: collection sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a + sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a url: "https://pub.dev" source: hosted version: "1.18.0" + version: "1.18.0" connectivity_plus: dependency: "direct main" description: @@ -612,18 +614,18 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a" + sha256: "3f87a60e8c63aecc975dda1ceedbc8f24de75f09e4856ea27daf8958f2f0ce05" url: "https://pub.dev" source: hosted - version: "10.0.4" + version: "10.0.5" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8" + sha256: "932549fb305594d82d7183ecd9fa93463e9914e1b67cacc34bc40906594a1806" url: "https://pub.dev" source: hosted - version: "3.0.3" + version: "3.0.5" leak_tracker_testing: dependency: transitive description: @@ -933,6 +935,7 @@ packages: description: flutter source: sdk version: "0.0.99" + version: "0.0.99" smithy: dependency: transitive description: @@ -986,9 +989,11 @@ packages: description: name: stack_trace sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" + sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" url: "https://pub.dev" source: hosted version: "1.11.1" + version: "1.11.1" stream_channel: dependency: transitive description: @@ -1010,9 +1015,11 @@ packages: description: name: string_scanner sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" + sha256: "556692adab6cfa87322a115640c11f13cb77b3f076ddcc5d6ae3c20242bedcde" url: "https://pub.dev" source: hosted version: "1.2.0" + version: "1.2.0" term_glyph: dependency: transitive description: @@ -1025,27 +1032,27 @@ packages: dependency: transitive description: name: test_api - sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f" + sha256: "5b8a98dafc4d5c4c9c72d8b31ab2b23fc13422348d2997120294d3bac86b4ddb" url: "https://pub.dev" source: hosted - version: "0.7.0" + version: "0.7.2" tflite_flutter: dependency: "direct main" description: name: tflite_flutter - sha256: "88b203c6efd7f4269aa137ef31e760d53e6639e60cd17bb86602462f72d4176b" + sha256: "48e6fde2ad97162bb66a16a142f4c4698add9e8cd397ce9d1cc7451b55537ac1" url: "https://pub.dev" source: hosted - version: "0.9.5" + version: "0.11.0" tflite_flutter_helper: dependency: "direct main" description: path: "." - ref: "43e87d4b9627539266dc20250beb35bf36320dce" - resolved-ref: "43e87d4b9627539266dc20250beb35bf36320dce" - url: "https://github.com/pnyompen/tflite_flutter_helper.git" + ref: "54a152256bc46fad8b2749061c7dd3bc5f2b63c3" + resolved-ref: "54a152256bc46fad8b2749061c7dd3bc5f2b63c3" + url: "https://github.com/VisualPT/tflite_flutter_helper.git" source: git - version: "0.3.0" + version: "0.4.1" tuple: dependency: transitive description: @@ -1082,10 +1089,10 @@ packages: dependency: transitive description: name: vm_service - sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec" + sha256: "5c5f338a667b4c644744b661f309fb8080bb94b18a7e91ef1dbd343bed00ed6d" url: "https://pub.dev" source: hosted - version: "14.2.1" + version: "14.2.5" watcher: dependency: transitive description: @@ -1114,10 +1121,10 @@ packages: dependency: transitive description: name: win32 - sha256: "0eaf06e3446824099858367950a813472af675116bf63f008a4c2a75ae13e9cb" + sha256: "68d1e89a91ed61ad9c370f9f8b6effed9ae5e0ede22a270bdfa6daf79fc2290a" url: "https://pub.dev" source: hosted - version: "5.5.0" + version: "5.5.4" win32_registry: dependency: transitive description: @@ -1159,5 +1166,5 @@ packages: source: hosted version: "3.1.2" sdks: - dart: ">=3.3.0 <4.0.0" - flutter: ">=3.22.0" + dart: ">=3.5.0 <4.0.0" + flutter: ">=3.24.0" diff --git a/pubspec.yaml b/pubspec.yaml index 7bd89c47..2c16c3c0 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -36,7 +36,7 @@ dependencies: amplify_storage_s3: ^2.4.1 app_tracking_transparency: ^2.0.6 bloc_concurrency: ^0.2.4 - camera: ^0.10.6 + camera: ^0.11.0+2 connectivity_plus: ^6.0.5 cupertino_icons: ^1.0.5 equatable: ^2.0.5 @@ -54,11 +54,11 @@ dependencies: printing: ^5.12.3 scidart: ^0.0.2-dev.12 sensors_plus: ^6.0.1 - tflite_flutter: ^0.9.5 + tflite_flutter: ^0.11.0 tflite_flutter_helper: git: - url: https://github.com/pnyompen/tflite_flutter_helper.git - ref: 43e87d4b9627539266dc20250beb35bf36320dce + url: https://github.com/VisualPT/tflite_flutter_helper.git + ref: 54a152256bc46fad8b2749061c7dd3bc5f2b63c3 vector_math: ^2.1.4 dev_dependencies: From af500963d46ed1e906c75d8608bd1c667400cce2 Mon Sep 17 00:00:00 2001 From: charlieforward9 Date: Tue, 15 Oct 2024 01:43:09 -0400 Subject: [PATCH 2/2] WIP feat: attempted `flutter_tflite`, `flutter_tflite_helper` bump --- .gitignore | 2 + ios/Podfile.lock | 38 ++++++---- lib/core/services/repository.dart | 2 +- lib/core/utils/image_utils.dart | 65 ++++++++++------- lib/features/assessment/config/dev/dev.dart | 2 +- .../inference/bloc/inference_bloc.dart | 11 ++- .../inference/bloc/inference_event.dart | 6 +- .../inference/constants/model_file.dart | 11 +-- lib/features/inference/service/ai_model.dart | 4 +- .../face_detection_service.dart | 17 ++--- .../service/face_mesh/face_mesh_service.dart | 15 +--- .../service/hands/hands_service.dart | 15 +--- .../service/model_inference_service.dart | 25 +++---- .../pose_class_service.dart | 12 +--- ...e_detection_classification_repository.dart | 7 +- .../pose_detection_service.dart | 69 ++++++++++++++----- .../view/pose_detection_indicator.dart | 2 +- .../service/video_detection_repository.dart | 33 +++++---- pubspec.lock | 30 ++++---- pubspec.yaml | 3 +- 20 files changed, 201 insertions(+), 168 deletions(-) diff --git a/.gitignore b/.gitignore index e0ffb25f..88e40555 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,11 @@ *.swp .DS_Store .atom/ +.build/ .buildlog/ .history .svn/ +.swiftpm/ migrate_working_dir/ # IntelliJ related diff --git a/ios/Podfile.lock b/ios/Podfile.lock index ed39ac4a..567901d8 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -48,16 +48,28 @@ PODS: - SQLite.swift/standard (= 0.13.2) - SQLite.swift/standard (0.13.2) - Starscream (4.0.4) - - TensorFlowLiteC (2.14.0): - - TensorFlowLiteC/Core (= 2.14.0) - - TensorFlowLiteC/Core (2.14.0) - - TensorFlowLiteSwift (2.14.0): - - TensorFlowLiteSwift/Core (= 2.14.0) - - TensorFlowLiteSwift/Core (2.14.0): - - TensorFlowLiteC (= 2.14.0) - - tflite_flutter (0.1.0): - - Flutter - - TensorFlowLiteSwift + - TensorFlowLiteC (2.12.0): + - TensorFlowLiteC/Core (= 2.12.0) + - TensorFlowLiteC/Core (2.12.0) + - TensorFlowLiteC/CoreML (2.12.0): + - TensorFlowLiteC/Core + - TensorFlowLiteC/Metal (2.12.0): + - TensorFlowLiteC/Core + - TensorFlowLiteSwift (2.12.0): + - TensorFlowLiteSwift/Core (= 2.12.0) + - TensorFlowLiteSwift/Core (2.12.0): + - TensorFlowLiteC (= 2.12.0) + - TensorFlowLiteSwift/CoreML (2.12.0): + - TensorFlowLiteC/CoreML (= 2.12.0) + - TensorFlowLiteSwift/Core (= 2.12.0) + - TensorFlowLiteSwift/Metal (2.12.0): + - TensorFlowLiteC/Metal (= 2.12.0) + - TensorFlowLiteSwift/Core (= 2.12.0) + - tflite_flutter (0.0.1): + - Flutter + - TensorFlowLiteSwift (= 2.12.0) + - TensorFlowLiteSwift/CoreML (= 2.12.0) + - TensorFlowLiteSwift/Metal (= 2.12.0) - tflite_flutter_helper (0.0.1): - Flutter @@ -133,9 +145,9 @@ SPEC CHECKSUMS: sensors_plus: 4b7c4bd9c9be2efbd575649368fe77ef09b429ad SQLite.swift: 4fc2be46c36392e3b87afe6fe7f1801c1daa07ef Starscream: 5178aed56b316f13fa3bc55694e583d35dd414d9 - TensorFlowLiteC: a836de9b6e6018665ccc349538bdd2c3b80dd7a5 - TensorFlowLiteSwift: f706b143b69d7bf433aba7b08049bdd11c321a10 - tflite_flutter: bf9c7d5c7ef011034c607d943d432623673c1c11 + TensorFlowLiteC: 20785a69299185a379ba9852b6625f00afd7984a + TensorFlowLiteSwift: 3a4928286e9e35bdd3e17970f48e53c80d25e793 + tflite_flutter: 9433d086a3060431bbc9f3c7c20d017db0e72d08 tflite_flutter_helper: 543b46b6bd064b21c92ea6e54bc0b29f1ce74cb5 PODFILE CHECKSUM: c238ed21013585bf5a2c0c98c3005ba48f3d16e2 diff --git a/lib/core/services/repository.dart b/lib/core/services/repository.dart index 1a02a4da..0daf9a86 100644 --- a/lib/core/services/repository.dart +++ b/lib/core/services/repository.dart @@ -21,7 +21,7 @@ abstract class Repository { } Future initialize(); - Future runPredict(String id, I input, Map? params); + Future runPredict(Map params); Future store( String id, I input, O output, Map? params); void reset() { diff --git a/lib/core/utils/image_utils.dart b/lib/core/utils/image_utils.dart index bcb3f403..f184cc34 100644 --- a/lib/core/utils/image_utils.dart +++ b/lib/core/utils/image_utils.dart @@ -1,7 +1,7 @@ import 'dart:isolate'; +import 'dart:typed_data'; import 'package:camera/camera.dart'; import 'package:image/image.dart' as image_lib; -import 'isolate_utils.dart'; // ignore: constant_identifier_names const IOS_BYTES_OFFSET = 28; @@ -9,6 +9,7 @@ const IOS_BYTES_OFFSET = 28; /// ImageUtils class ImageUtils { /// Converts a [CameraImage] in YUV420 or BGRA8888 format to [Image] in RGB format + /// Ref: https://gist.github.com/Alby-o/fe87e35bc21d534c8220aed7df028e03?permalink_comment_id=4574528#gistcomment-4574528 static image_lib.Image convertCameraImage(CameraImage cameraImage) { return switch (cameraImage.format.group) { ImageFormatGroup.yuv420 => convertYUV420ToImage(cameraImage), @@ -23,14 +24,34 @@ class ImageUtils { static image_lib.Image convertBGRA8888ToImage(CameraImage cameraImage) { final plane = cameraImage.planes[0]; - return image_lib.Image.fromBytes( + // Calculate the number of pixels + int numPixels = cameraImage.width * cameraImage.height; + + // Create a Float32List for the RGB image without the alpha channel + Float32List floatList = Float32List(numPixels * 3); // 3 channels (RGB) + + // Populate the floatList by ignoring the alpha channel + for (int i = 0, j = 0; i < plane.bytes.length; i += 4) { + floatList[j++] = plane.bytes[i] / 255.0; // Red + floatList[j++] = plane.bytes[i + 1] / 255.0; // Green + floatList[j++] = plane.bytes[i + 2] / 255.0; // Blue + // Skip alpha (i + 3) + } + + // Determine the rowStride for the float32 image + int floatRowStride = plane.bytesPerRow * 3; // Adjust for float32 bytes + + final img = image_lib.Image.fromBytes( width: cameraImage.width, height: cameraImage.height, - bytes: plane.bytes.buffer, - rowStride: plane.bytesPerRow, - bytesOffset: IOS_BYTES_OFFSET, - order: image_lib.ChannelOrder.bgra, + bytes: floatList.buffer, + rowStride: floatRowStride, + // bytesOffset: IOS_BYTES_OFFSET, + order: image_lib.ChannelOrder.rgb, + format: image_lib.Format.float32, ); + return image_lib.copyResize(img, + width: 256, height: 256, interpolation: image_lib.Interpolation.linear); } /// Converts a [CameraImage] in YUV420 format to [image_lib.Image] in RGB format @@ -106,22 +127,20 @@ class ImageUtils { // } /// Convert a [Image] to PNG - static Future> convert( - {required IsolateUtils isolateUtils, required CameraImage image}) async { - final responsePort = ReceivePort(); - - toPng(Map params) => - image_lib.encodePng(ImageUtils.convertCameraImage(params['image'])); - - isolateUtils.sendMessage( - handler: toPng, - params: { - 'image': image, - }, - sendPort: isolateUtils.sendPort, - responsePort: responsePort, - ); - - return await responsePort.first; + static Future> convert({required CameraImage image}) async { + //final responsePort = ReceivePort(); + + final encodedImage = Isolate.run( + () => image_lib.encodePng(ImageUtils.convertCameraImage(image))); + // isolateUtils.sendMessage( + // handler: toPng, + // params: { + // 'image': image, + // }, + // sendPort: isolateUtils.sendPort, + // responsePort: responsePort, + // ); + + return await encodedImage; } } diff --git a/lib/features/assessment/config/dev/dev.dart b/lib/features/assessment/config/dev/dev.dart index d807ad00..7319217a 100644 --- a/lib/features/assessment/config/dev/dev.dart +++ b/lib/features/assessment/config/dev/dev.dart @@ -27,7 +27,7 @@ class DEV implements BaseAssessment { int get trials => 1; @override - int get trialDuration => 5; + int get trialDuration => 500; @override List get conditionNames => [ diff --git a/lib/features/inference/bloc/inference_bloc.dart b/lib/features/inference/bloc/inference_bloc.dart index 8a39562b..77d77c59 100644 --- a/lib/features/inference/bloc/inference_bloc.dart +++ b/lib/features/inference/bloc/inference_bloc.dart @@ -62,8 +62,13 @@ class InferenceBloc } if (processing) return null; processing = true; - final data = await inferenceRepository.runPredict( - event.id, event.input, event.params); + final params = { + "id": event.id, + "input": event.input, + "params": event.params, + }; + + final data = await inferenceRepository.runPredict(params); processing = false; if (isReady && !isClosed && !emit.isDone) { add(InferenceOutput( @@ -121,6 +126,6 @@ class InferenceBloc void onError(Object error, StackTrace stackTrace) { // Always call super.onError with the current error super.onError(error, stackTrace); - log('InferenceBloc Error: $stackTrace $error'); + log('InferenceBloc Error: $error $stackTrace'); } } diff --git a/lib/features/inference/bloc/inference_event.dart b/lib/features/inference/bloc/inference_event.dart index 262ba136..cfa23284 100644 --- a/lib/features/inference/bloc/inference_event.dart +++ b/lib/features/inference/bloc/inference_event.dart @@ -17,7 +17,7 @@ class InferenceInit extends InferenceEvent { } class InferenceStart extends InferenceEvent { - final Map? params; + final Map params; const InferenceStart(super.id, this.params); @override @@ -26,11 +26,11 @@ class InferenceStart extends InferenceEvent { class InferenceInput extends InferenceEvent { final I input; - final Map? params; + final Map params; const InferenceInput( super.id, { required this.input, - this.params, + required this.params, }); @override diff --git a/lib/features/inference/constants/model_file.dart b/lib/features/inference/constants/model_file.dart index 229abc04..cea560f4 100644 --- a/lib/features/inference/constants/model_file.dart +++ b/lib/features/inference/constants/model_file.dart @@ -1,10 +1,11 @@ mixin ModelFile { - static const String faceDetection = 'models/face_detection.tflite'; - static const String faceMesh = 'models/face_landmark.tflite'; - static const String hands = 'models/hand_landmark.tflite'; - static const String poseDetection = 'models/pose_landmark_heavy.tflite'; + static const String faceDetection = 'assets/models/face_detection.tflite'; + static const String faceMesh = 'assets/models/face_landmark.tflite'; + static const String hands = 'assets/models/hand_landmark.tflite'; + static const String poseDetection = + 'assets/models/pose_landmark_heavy.tflite'; static const String poseClassification = - 'models/pose_classification.tflite'; //TODO: + 'assets/models/pose_classification.tflite'; //TODO: } enum InferenceType { diff --git a/lib/features/inference/service/ai_model.dart b/lib/features/inference/service/ai_model.dart index 599c9ccb..bbf988a3 100644 --- a/lib/features/inference/service/ai_model.dart +++ b/lib/features/inference/service/ai_model.dart @@ -4,7 +4,7 @@ import 'package:visualpt/features/inference/bloc/inference_bloc.dart'; // ignore: must_be_immutable abstract class AiModel extends Equatable { - AiModel({this.interpreter}); + AiModel(); final inputShapes = >[]; final inputTypes = []; @@ -19,7 +19,7 @@ abstract class AiModel extends Equatable { int get getAddress; Future loadModel(); - O? predict(String id, I input, Map? params); + O? predict(Map params); } abstract class AiModelOutput extends Identifiable { diff --git a/lib/features/inference/service/face_detection/face_detection_service.dart b/lib/features/inference/service/face_detection/face_detection_service.dart index 8622b3b4..adf851fe 100644 --- a/lib/features/inference/service/face_detection/face_detection_service.dart +++ b/lib/features/inference/service/face_detection/face_detection_service.dart @@ -7,7 +7,6 @@ import 'package:tflite_flutter/tflite_flutter.dart'; import 'package:tflite_flutter_helper/tflite_flutter_helper.dart'; import '../../constants/model_file.dart'; -import '../../../../core/utils/image_utils.dart'; import '../ai_model.dart'; import 'anchors.dart'; import 'generate_anchors.dart'; @@ -85,13 +84,15 @@ class FaceDetection extends AiModel { } @override - FaceBboxOutput? predict( - String id, image_lib.Image input, Map? params) { + FaceBboxOutput? predict(Map params) { if (interpreter == null) { log('Interpreter not initialized'); return null; } + final String id = params['id']; + image_lib.Image input = params['input']; + final options = OptionsFace( numClasses: 1, numBoxes: 896, @@ -167,16 +168,6 @@ class FaceDetection extends AiModel { } } -FaceBboxOutput? runFaceDetector(Map params) { - final faceDetection = FaceDetection( - interpreter: Interpreter.fromAddress(params['detectorAddress'])); - final id = params['id'] as String; - final input = ImageUtils.convertCameraImage(params['input']); - final result = faceDetection.predict(id, input, {}); - - return result; -} - class FaceBboxOutput extends AiModelOutput { const FaceBboxOutput(super.id, super.output); diff --git a/lib/features/inference/service/face_mesh/face_mesh_service.dart b/lib/features/inference/service/face_mesh/face_mesh_service.dart index 571d839c..868d6308 100644 --- a/lib/features/inference/service/face_mesh/face_mesh_service.dart +++ b/lib/features/inference/service/face_mesh/face_mesh_service.dart @@ -7,7 +7,6 @@ import 'package:tflite_flutter/tflite_flutter.dart'; import 'package:tflite_flutter_helper/tflite_flutter_helper.dart'; import '../../constants/model_file.dart'; -import '../../../../core/utils/image_utils.dart'; import '../ai_model.dart'; // ignore: must_be_immutable @@ -57,12 +56,13 @@ class FaceMesh extends AiModel { } @override - FaceOutput? predict( - String id, image_lib.Image input, Map? params) { + FaceOutput? predict(Map params) { if (interpreter == null) { log('Interpreter not initialized'); return null; } + final String id = params['id']; + image_lib.Image input = params['input']; if (Platform.isAndroid) { input = image_lib.copyRotate(input, angle: -90); @@ -100,15 +100,6 @@ class FaceMesh extends AiModel { } } -FaceOutput? runFaceMesh(Map params) { - final faceMesh = - FaceMesh(interpreter: Interpreter.fromAddress(params['detectorAddress'])); - final image = ImageUtils.convertCameraImage(params['input']); - final result = faceMesh.predict(params["id"], image, {}); - - return result; -} - class FaceOutput extends AiModelOutput { const FaceOutput(super.id, super.output); diff --git a/lib/features/inference/service/hands/hands_service.dart b/lib/features/inference/service/hands/hands_service.dart index 49ae1ea9..842aede1 100644 --- a/lib/features/inference/service/hands/hands_service.dart +++ b/lib/features/inference/service/hands/hands_service.dart @@ -7,7 +7,6 @@ import 'package:tflite_flutter/tflite_flutter.dart'; import 'package:tflite_flutter_helper/tflite_flutter_helper.dart'; import '../../constants/model_file.dart'; -import '../../../../core/utils/image_utils.dart'; import '../ai_model.dart'; // ignore: must_be_immutable @@ -59,12 +58,13 @@ class Hands extends AiModel { } @override - HandsOutput? predict( - String id, image_lib.Image input, Map? params) { + HandsOutput? predict(Map params) { if (interpreter == null) { log('Interpreter not initialized'); return null; } + final String id = params['id']; + image_lib.Image input = params['input']; if (Platform.isAndroid) { input = image_lib.copyRotate(input, angle: -90); @@ -106,15 +106,6 @@ class Hands extends AiModel { } } -HandsOutput? runHandDetector(Map params) { - final hands = - Hands(interpreter: Interpreter.fromAddress(params['detectorAddress'])); - final image = ImageUtils.convertCameraImage(params['input']); - final result = hands.predict(params["id"], image, {}); - - return result; -} - class HandsOutput extends AiModelOutput { const HandsOutput(super.id, super.output); diff --git a/lib/features/inference/service/model_inference_service.dart b/lib/features/inference/service/model_inference_service.dart index b8ba18a1..d063640d 100644 --- a/lib/features/inference/service/model_inference_service.dart +++ b/lib/features/inference/service/model_inference_service.dart @@ -20,30 +20,25 @@ enum Models { class ModelInferenceService { late AiModel model; - late Function handler; + late AiModelOutput? Function( + Map params, + ) handler; Future inference({ - required String id, required IsolateUtils isolateUtils, - required I input, - required Map? params, + required Map params, }) async { final responsePort = ReceivePort(); - + // print("Sending $handler, with params: $params"); isolateUtils.sendMessage( handler: handler, - params: { - 'id': id, - 'input': input, - 'detectorAddress': model.getAddress, - 'params': params, - }, + params: params, sendPort: isolateUtils.sendPort, responsePort: responsePort, ); final results = await responsePort.first; - responsePort.close(); + //responsePort.close(); return results; } @@ -52,27 +47,23 @@ class ModelInferenceService { case Models.FaceDetection: log("Face Detection model configured"); model = locator(); - handler = runFaceDetector; break; case Models.FaceMesh: log("Face Mesh model configured"); model = locator(); - handler = runFaceMesh; break; case Models.Hands: log("Hands model configured"); model = locator(); - handler = runHandDetector; break; case Models.PoseDetect: log("Pose detection model configured"); model = locator(); - handler = runPoseDetector; break; case Models.PoseClass: log("Pose classification model configured"); model = locator(); - handler = runPoseClassifier; } + handler = model.predict; } } diff --git a/lib/features/inference/service/pose_classification/pose_class_service.dart b/lib/features/inference/service/pose_classification/pose_class_service.dart index 8cc5c3be..25f8af97 100644 --- a/lib/features/inference/service/pose_classification/pose_class_service.dart +++ b/lib/features/inference/service/pose_classification/pose_class_service.dart @@ -47,7 +47,7 @@ class NormConst { class PoseClassification extends AiModel { - PoseClassification({super.interpreter}) { + PoseClassification() { loadModel(); } // static const int inputSize = 256; @@ -84,12 +84,12 @@ class PoseClassification } @override - PoseClassificationOutput? predict( - String id, PoseDetectionOutput input, Map? params) { + PoseClassificationOutput? predict(Map params) { if (interpreter == null) { log('Interpreter not initialized'); return null; } + final PoseDetectionOutput input = params['input']; final List> data = input.output; final processedInput = data.sublist(0, 33).map((e) => e.sublist(0, 5)).toList(); @@ -109,12 +109,6 @@ class PoseClassification } } -PoseClassificationOutput? runPoseClassifier(Map params) { - final pose = PoseClassification( - interpreter: Interpreter.fromAddress(params['detectorAddress'])); - return pose.predict(params['id'], params['input'], params['params']); -} - List>? normalizeDetection(List> dataPoint) { // Assuming 'dataPoint' is a List> where each inner list has 5 elements. if (dataPoint.length < 25) { diff --git a/lib/features/inference/service/pose_detection/pose_detection_classification_repository.dart b/lib/features/inference/service/pose_detection/pose_detection_classification_repository.dart index cd0313a0..34968c11 100644 --- a/lib/features/inference/service/pose_detection/pose_detection_classification_repository.dart +++ b/lib/features/inference/service/pose_detection/pose_detection_classification_repository.dart @@ -22,15 +22,14 @@ class DetectionClassificationRepository } @override - Future runPredict(String id, - PoseDetectionOutput input, Map? params) async { + Future runPredict( + Map params) async { if (!isInitialized) { throw Exception("Classification Inference Service not initialized"); } + final input = params['input'] as PoseDetectionOutput; final output = await _poseClassService.inference( - id: id, isolateUtils: _classificationIsolateUtils, - input: input, params: params, ) as PoseClassificationOutput?; if (output != null) localStorage[input] = output; diff --git a/lib/features/inference/service/pose_detection/pose_detection_service.dart b/lib/features/inference/service/pose_detection/pose_detection_service.dart index 61569937..d1af8fea 100644 --- a/lib/features/inference/service/pose_detection/pose_detection_service.dart +++ b/lib/features/inference/service/pose_detection/pose_detection_service.dart @@ -9,7 +9,7 @@ import '../ai_model.dart'; // ignore: must_be_immutable class PoseDetection extends AiModel { - PoseDetection({super.interpreter}) { + PoseDetection() { loadModel(); } static const int inputSize = 256; @@ -29,6 +29,13 @@ class PoseDetection extends AiModel { interpreter ??= await Interpreter.fromAsset(ModelFile.poseDetection, options: interpreterOptions); + final inputTensors = interpreter!.getInputTensors(); + print(inputTensors.toString()); + for (var tensor in inputTensors) { + inputShapes.add(tensor.shape); + inputTypes.add(tensor.type); + } + final outputTensors = interpreter!.getOutputTensors(); for (var tensor in outputTensors) { @@ -41,13 +48,15 @@ class PoseDetection extends AiModel { } @override - PoseDetectionOutput? predict( - String id, TensorImage input, Map? params) { + PoseDetectionOutput? predict(Map params) { if (interpreter == null) { log('Interpreter not initialized'); return null; } + String id = params['id']; + TensorImage input = params['input']; + // if (Platform.isAndroid) { // image = image_lib.copyRotate(image, -90); // image = image_lib.flipHorizontal(image); @@ -70,10 +79,9 @@ class PoseDetection extends AiModel { }; interpreter!.runForMultipleInputs(inputs, outputs); - - if (outputIdentity1.getDoubleValue(0) < threshold) { - return null; - } + // if (outputIdentity1.getDoubleValue(0) < threshold) { + // return null; + // } // Data Shape Defined by: https://storage.googleapis.com/mediapipe-assets/Model%20Card%20BlazePose%20GHUM%203D.pdf // Further detail here: https://github.com/google/mediapipe/issues/4752 @@ -81,18 +89,43 @@ class PoseDetection extends AiModel { .getDoubleList() .reshape([39, 5]) as List>; + // Save the input TensorImage to a file with landmark points drawn + // saveTensorImageWithLandmarks( + // input, landmarkPoints, 'input_image_with_landmarks.png'); + return PoseDetectionOutput(id, landmarkPoints); } -} - -PoseDetectionOutput? runPoseDetector(Map params) { - final pose = PoseDetection( - interpreter: Interpreter.fromAddress(params['detectorAddress'])); - - final id = params['id'] as String; - final input = params['input'] as TensorImage; - - final result = pose.predict(id, input, params['params']); - return result; + // // Utility function to save TensorImage with landmarks drawn on it + // void saveTensorImageWithLandmarks( + // TensorImage tensorImage, List> landmarks, String filePath) { + // // Convert the TensorImage to a standard image (image_lib.Image) + // final image = tensorImage + // .image; // Assuming TensorImage can directly provide an image_lib.Image + + // // Draw the landmark points on the image + // for (var landmark in landmarks) { + // final x = (landmark[0] * image.width).toInt(); // Scale X to image size + // final y = (landmark[1] * image.height).toInt(); // Scale Y to image size + + // // Draw a small circle at each landmark point + // drawCircle(image, + // x: x, + // y: y, + // radius: 4, + // color: ColorUint8.rgb(255, 0, 0)); // Red circles + // } + + // // Convert the image with landmarks to PNG bytes + // final pngBytes = encodePng(image); + + // // Get the full path to the repository's testimage/ directory + // final fullPath = path.join(Directory.current.path, filePath); + + // // Create the file and write the PNG bytes to it + // File(fullPath).createSync(recursive: true); + // File(fullPath).writeAsBytesSync(pngBytes); + + // print('Input image with landmarks saved to: $fullPath'); + // } } diff --git a/lib/features/inference/view/pose_detection_indicator.dart b/lib/features/inference/view/pose_detection_indicator.dart index c9f5fa8e..f8662d3a 100644 --- a/lib/features/inference/view/pose_detection_indicator.dart +++ b/lib/features/inference/view/pose_detection_indicator.dart @@ -27,7 +27,7 @@ class DeviceCapture extends Identifiable { String toString() => "DeviceCapture{id: $id}"; @override //the props - List get props => [id]; + List get props => [id, input, measurements]; } class PoseDetectionIndicator extends StatelessWidget { diff --git a/lib/features/video/service/video_detection_repository.dart b/lib/features/video/service/video_detection_repository.dart index 75451e1b..fb005fef 100644 --- a/lib/features/video/service/video_detection_repository.dart +++ b/lib/features/video/service/video_detection_repository.dart @@ -20,7 +20,6 @@ class VideoDetectionRepository final ModelInferenceService _poseDetectionService = ModelInferenceService(); final IsolateUtils _detectionIsolateUtils = IsolateUtils(); - final IsolateUtils _convertIsolateUtils = IsolateUtils(); final accelStream = userAccelerometerEventStream(); final gyroStream = gyroscopeEventStream(); StreamSubscription? _gyroSubscription; @@ -45,24 +44,27 @@ class VideoDetectionRepository @override Future initialize() async { await _detectionIsolateUtils.initIsolate("Pose Detection Isolate"); - await _convertIsolateUtils.initIsolate("Image Conversion Isolate"); _poseDetectionService.setModelConfig(3); isInitialized = true; } @override - Future runPredict( - String id, DeviceCapture input, Map? params) async { + Future runPredict(Map params) async { if (!isInitialized) { throw Exception( "runPredict: Detection Inference Service not initialized"); } + final id = params['id'] as String; + final input = params['input'] as DeviceCapture; final data = preProcessData(input.input); + final newParams = { + 'id': id, + 'input': data, + 'params': params['params'], + }; final output = await _poseDetectionService.inference( - id: id, isolateUtils: _detectionIsolateUtils, - input: data, - params: params, + params: newParams, ) as PoseDetectionOutput?; if (output != null) localStorage[input] = output; return output; @@ -77,7 +79,7 @@ class VideoDetectionRepository StorageService storageService = StorageService(); final png = await ImageUtils.convert( - isolateUtils: _convertIsolateUtils, + // isolateUtils: _convertIsolateUtils, image: input.input, ); final data = VPTDetection( @@ -103,7 +105,6 @@ class VideoDetectionRepository @override void dispose() { _detectionIsolateUtils.dispose(); - _convertIsolateUtils.dispose(); _gyroSubscription?.cancel(); _accelSubscription?.cancel(); } @@ -118,15 +119,19 @@ class VideoDetectionRepository } } +//TODO: FIGURE OUT WHY THIS SHIT IS BROKEN TensorImage preProcessData(CameraImage input) { final image = ImageUtils.convertCameraImage(input); var tensorImage = TensorImage(TensorType.float32); tensorImage.loadImage(image); - final imageProcessor = ImageProcessorBuilder() - .add(ResizeOp(256, 256, ResizeMethod.BILINEAR)) - .add(NormalizeOp(0, 255)) - .build(); - return imageProcessor.process(tensorImage); + // final imageProcessor = ImageProcessorBuilder() + // .add(ResizeOp(256, 256, ResizeMethod.BILINEAR)) + // .add(NormalizeOp(0, 255)) + // .build(); + + // final output = imageProcessor.process(tensorImage); + + return tensorImage; } diff --git a/pubspec.lock b/pubspec.lock index 04ca9c0c..80c01b9d 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -269,10 +269,10 @@ packages: dependency: transitive description: name: camera_android_camerax - sha256: e3627fdc2132d89212b8a8676679f5b07008c7e3d8ae00cea775c3397f9e742b + sha256: "011be2ab0e5b3e3aa8094413fa890f8c5c5afd7cfdaef353a992047d4dab5780" url: "https://pub.dev" source: hosted - version: "0.6.10" + version: "0.6.8+2" camera_avfoundation: dependency: transitive description: @@ -614,18 +614,18 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "3f87a60e8c63aecc975dda1ceedbc8f24de75f09e4856ea27daf8958f2f0ce05" + sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a" url: "https://pub.dev" source: hosted - version: "10.0.5" + version: "10.0.4" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: "932549fb305594d82d7183ecd9fa93463e9914e1b67cacc34bc40906594a1806" + sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8" url: "https://pub.dev" source: hosted - version: "3.0.5" + version: "3.0.3" leak_tracker_testing: dependency: transitive description: @@ -1032,10 +1032,10 @@ packages: dependency: transitive description: name: test_api - sha256: "5b8a98dafc4d5c4c9c72d8b31ab2b23fc13422348d2997120294d3bac86b4ddb" + sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f" url: "https://pub.dev" source: hosted - version: "0.7.2" + version: "0.7.0" tflite_flutter: dependency: "direct main" description: @@ -1048,11 +1048,11 @@ packages: dependency: "direct main" description: path: "." - ref: "54a152256bc46fad8b2749061c7dd3bc5f2b63c3" - resolved-ref: "54a152256bc46fad8b2749061c7dd3bc5f2b63c3" + ref: "949a6f8ca0c8b3b686864727c88cb1c58f80dff0" + resolved-ref: "949a6f8ca0c8b3b686864727c88cb1c58f80dff0" url: "https://github.com/VisualPT/tflite_flutter_helper.git" source: git - version: "0.4.1" + version: "0.4.3" tuple: dependency: transitive description: @@ -1089,10 +1089,10 @@ packages: dependency: transitive description: name: vm_service - sha256: "5c5f338a667b4c644744b661f309fb8080bb94b18a7e91ef1dbd343bed00ed6d" + sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec" url: "https://pub.dev" source: hosted - version: "14.2.5" + version: "14.2.1" watcher: dependency: transitive description: @@ -1166,5 +1166,5 @@ packages: source: hosted version: "3.1.2" sdks: - dart: ">=3.5.0 <4.0.0" - flutter: ">=3.24.0" + dart: ">=3.4.0 <4.0.0" + flutter: ">=3.22.0" diff --git a/pubspec.yaml b/pubspec.yaml index 2c16c3c0..ea87125a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -50,7 +50,6 @@ dependencies: intl: ^0.19.0 path_provider: ^2.1.4 pdf: ^3.11.1 - # pdf_widget_wrapper: ^1.0.4 # This is for a build error on 1.0.3, when the dependent package has an available update, this can be removed printing: ^5.12.3 scidart: ^0.0.2-dev.12 sensors_plus: ^6.0.1 @@ -58,7 +57,7 @@ dependencies: tflite_flutter_helper: git: url: https://github.com/VisualPT/tflite_flutter_helper.git - ref: 54a152256bc46fad8b2749061c7dd3bc5f2b63c3 + ref: 949a6f8ca0c8b3b686864727c88cb1c58f80dff0 vector_math: ^2.1.4 dev_dependencies: