Analyze Receipts with LLMs

functions/.gitignore

@@ -7,4 +7,5 @@ typings/
 # Node.js dependency directory
 node_modules/
 
-serviceAccount.json
+serviceAccount.json
+.env

functions/index.ts

@@ -1,24 +1,25 @@
 import * as admin from 'firebase-admin'
 import 'firebase-functions'
 import * as functionsV1 from 'firebase-functions/v1'
 import {
   onDocumentCreated,
   onDocumentUpdated,
   onDocumentWritten,
 } from 'firebase-functions/v2/firestore'
 import { onCall } from 'firebase-functions/v2/https'
-import { analyzeReceipt } from './src/functions/analyzeReceipt'
+import { analyzeReceipt } from './src/functions/receiptAnalysis/analyzeReceipt'
 import { deleteRelatedGroupData } from './src/functions/deleteRelatedGroupData'
 import { updateUserExpenses } from './src/functions/docSync/updateUserExpenses'
 import { updateUserGroupsWhenExpenseChanges } from './src/functions/docSync/updateUserGroupsWhenExpenseChanges'
 import { updateUserGroupsWhenGroupChanges } from './src/functions/docSync/updateUserGroupsWhenGroupChanges'
 import { getLatestRelease } from './src/functions/getLatestRelease'
 import { notifyWhenExpenseCreated } from './src/functions/notifications/notifyWhenExpenseCreated'
 import { notifyWhenExpenseUpdated } from './src/functions/notifications/notifyWhenExpenseUpdated'
 import { notifyWhenGroupDebtThresholdReached } from './src/functions/notifications/notifyWhenGroupDebtThresholdReached'
 import { removeUserFromGroups } from './src/functions/userManagement/removeUserFromGroups'
 import { updateUser } from './src/functions/userManagement/updateUser'
 import { UserData } from './src/types/userData'
+import { analyzeReceiptWithAI } from './src/functions/receiptAnalysis/analyzeReceiptWithAI'
 require('firebase-functions/logger/compat')
 
 admin.initializeApp()
@@ -34,10 +35,7 @@
 )
 
 export const getReceiptData = onCall(
-  {
-    timeoutSeconds: 300,
-    memory: '4GiB',
-  },
+  { timeoutSeconds: 300, memory: '4GiB' },
   async (event) => {
     if (!event.data.receiptUrl) {
       throw new Error('The parameter receiptUrl is required.')
@@ -51,6 +49,17 @@
   }
 )
 
+export const getReceiptDataWithAI = onCall(
+  { timeoutSeconds: 60 },
+  async (event) => {
+    if (!event.data.receiptUrl) {
+      throw new Error('The parameter receiptUrl is required.')
+    }
+
+    return analyzeReceiptWithAI(event.data.receiptUrl)
+  }
+)
+
 export const cleanUpOnAccountDeletion = functionsV1.auth
   .user()
   .onDelete(async (user, _) => {

functions/package.json

@@ -9,8 +9,9 @@
     "test": "export FIREBASE_AUTH_EMULATOR_HOST=\"localhost:9099\" && export FIRESTORE_EMULATOR_HOST=\"localhost:8080\" && jest",
     "lint": "eslint --ext .ts . --fix",
     "format": "prettier --write \"src/{**/*,*}.{js,ts,jsx,tsx,json}\"",
-    "build": "tsc",
-    "listen": "tsc --watch",
+    "copy-assets": "mkdir -p lib/src/assets && cp -r src/assets/* lib/src/assets/",
+    "build": "tsc && npm run copy-assets",
+    "listen": "npm run copy-assets && tsc --watch",
     "serve": "npm run build && firebase emulators:start --only functions",
     "shell": "npm run build && firebase functions:shell",
     "start": "npm run shell",
@@ -24,7 +25,8 @@
     "firebase-admin": "^12.1.1",
     "firebase-functions": "^7.0.0",
     "lodash": "^4.17.21",
-    "node-fetch": "^3.3.2"
+    "node-fetch": "^3.3.2",
+    "openai": "^6.41.0"
   },
   "devDependencies": {
     "@types/jest": "^29",

functions/src/assets/prompt.md

@@ -0,0 +1,12 @@
+Analyze the receipt image at the following URL and extract the list of products, their prices, and quantities. 
+Return the data in JSON format with the following structure: 
+```
+[{ 
+    "name": string, 
+    "value": number, 
+}]
+```
+
+**Important rules**
+- respond with JSON only
+- if the image does not contain a receipt, response with `[]`

functions/src/functions/analyzeReceipt.ts → functions/src/functions/receiptAnalysis/analyzeReceipt.ts

@@ -1,10 +1,10 @@
 import { ImageAnnotatorClient } from '@google-cloud/vision'
 import { google } from '@google-cloud/vision/build/protos/protos'
 import { max, min } from 'lodash'
-import { BoxWithText, RowOfText } from '../types/geometry'
-import { Product } from '../types/products'
-import { defaultStore, StoreName, stores } from '../types/stores'
-import { distanceToRow, height, toBoxWithText } from '../utils/geometryUtils'
+import { BoxWithText, RowOfText } from '../../types/geometry'
+import { Product } from '../../types/products'
+import { defaultStore, StoreName, stores } from '../../types/stores'
+import { distanceToRow, height, toBoxWithText } from '../../utils/geometryUtils'
 import fetch from 'node-fetch'
 
 type IEntityAnnotation = google.cloud.vision.v1.IEntityAnnotation

functions/src/functions/receiptAnalysis/analyzeReceiptWithAI.ts

@@ -0,0 +1,32 @@
+import OpenAI from 'openai'
+import { Product } from '../../types/products'
+import { readFile } from 'node:fs/promises'
+import path from 'node:path'
+
+export async function analyzeReceiptWithAI(
+  receiptUrl: string
+): Promise<Product[]> {
+  console.log(`Analyzing receipt at ${receiptUrl}`)
+
+  const prompt = await readFile(
+    path.join(__dirname, '../../assets/prompt.md'),
+    'utf-8'
+  )
+
+  const client = new OpenAI()
+  const response = await client.responses.create({
+    model: 'gpt-5-nano',
+    input: [
+      {
+        role: 'user',
+        content: [
+          { type: 'input_text', text: prompt },
+          { type: 'input_image', image_url: receiptUrl, detail: 'auto' },
+        ],
+      },
+    ],
+  })
+
+  const products = JSON.parse(response.output_text) as Product[]
+  return products
+}

functions/tests/functions/analyzeReceipt.test.ts

@@ -1,7 +1,7 @@
 import {
   analyzeReceipt,
   IAnnotateResponse,
-} from '../../src/functions/analyzeReceipt'
+} from '../../src/functions/receiptAnalysis/analyzeReceipt'
 import lcboReceiptShortData from '../__stubs__/receipt_data/lcbo/short/data.json'
 import walmartReceiptLongData from '../__stubs__/receipt_data/walmart/long/data.json'
 import walmartReceiptMediumData from '../__stubs__/receipt_data/walmart/medium/data.json'
@@ -14,7 +14,7 @@ import { StoreName } from '../../src/types/stores'
 
 const documentTextDetection = jest.fn()
 jest.mock('@google-cloud/vision', () => ({
-  ImageAnnotatorClient: jest.fn().mockImplementation(function() {
+  ImageAnnotatorClient: jest.fn().mockImplementation(function () {
     return { documentTextDetection }
   }),
 }))

lib/data/services/callables.dart

@@ -5,11 +5,14 @@ import 'package:statera/data/models/models.dart';
 
 @GenerateNiceMocks([MockSpec<Callables>()])
 class Callables {
-  static HttpsCallable _getReceiptData =
-      FirebaseFunctions.instance.httpsCallable('getReceiptData');
+  static HttpsCallable _getReceiptData = FirebaseFunctions.instance
+      .httpsCallable('getReceiptData');
 
-  static HttpsCallable _getLatestVersion =
-      FirebaseFunctions.instance.httpsCallable('getLatestAppVersion');
+  static HttpsCallable _getReceiptDataWithAI = FirebaseFunctions.instance
+      .httpsCallable('getReceiptDataWithAI');
+
+  static HttpsCallable _getLatestVersion = FirebaseFunctions.instance
+      .httpsCallable('getLatestAppVersion');
 
   Future<List<Item>> getReceiptData({
     required String receiptUrl,
@@ -19,7 +22,7 @@ class Callables {
     var response = await _getReceiptData({
       'receiptUrl': receiptUrl,
       'storeName': selectedStore,
-      'withNameImprovement': withNameImprovement
+      'withNameImprovement': withNameImprovement,
     });
 
     return (response.data as List<dynamic>).map((itemData) {
@@ -33,6 +36,20 @@ class Callables {
     }).toList();
   }
 
+  Future<List<Item>> getReceiptDataWithAI({required String receiptUrl}) async {
+    var response = await _getReceiptDataWithAI({'receiptUrl': receiptUrl});
+
+    return (response.data as List<dynamic>).map((itemData) {
+      final value = double.tryParse(itemData['value'].toString()) ?? 0;
+      final quantity = int.tryParse(itemData['quantity'].toString()) ?? 1;
+      return SimpleItem(
+        name: itemData['name'] ?? 'item',
+        value: value,
+        partition: quantity,
+      );
+    }).toList();
+  }
+
   static Future<Version> getLatestAndroidVersion() async {
     var response = await _getLatestVersion({'platform': 'android'});
     return Version.fromString(response.data);

lib/data/services/feature_service.dart

@@ -7,4 +7,7 @@ import 'package:mockito/annotations.dart';
 class FeatureService {
   bool get debtRedirectionEnabled =>
       FirebaseRemoteConfig.instance.getBool('redirect_debt_feature_flag');
+
+  bool get aiReceiptAnalysisEnabled =>
+      FirebaseRemoteConfig.instance.getBool('ai_receipt_analysis_feature_flag');
 }

lib/main.dart

@@ -27,20 +27,19 @@ Future<void> main() async {
   await FirebaseRemoteConfig.instance.setDefaults(<String, dynamic>{
     'greeting_message': 'Welcome to Statera!',
     'show_greeting_dialog': false,
-    'redirect_debt_feature_flag': true
+    'redirect_debt_feature_flag': true,
+    'ai_receipt_analysis_feature_flag': false,
   });
 
   configureEmulators();
 
   final initialDynamicLink = null;
   final dynamicLinkPath = initialDynamicLink?.link.path;
-  final initialNotificationMessage =
-      await FirebaseMessaging.instance.getInitialMessage();
+  final initialNotificationMessage = await FirebaseMessaging.instance
+      .getInitialMessage();
   final notificationPath = AppLaunchHandler.getPath(initialNotificationMessage);
 
-  runApp(Statera(
-    initialRoute: dynamicLinkPath ?? notificationPath,
-  ));
+  runApp(Statera(initialRoute: dynamicLinkPath ?? notificationPath));
 }
 
 class Statera extends StatelessWidget {
@@ -57,16 +56,18 @@ class Statera extends StatelessWidget {
         child: CustomThemeBuilder(
           builder: (lightTheme, darkTheme) {
             return CustomLayoutBuilder(
-              child: Builder(builder: (context) {
-                return MaterialApp.router(
-                  title: kAppName,
-                  theme: lightTheme,
-                  darkTheme: darkTheme,
-                  themeMode: ThemeMode.system,
-                  routerConfig: CustomRouterConfig.create(context),
-                  debugShowCheckedModeBanner: false,
-                );
-              }),
+              child: Builder(
+                builder: (context) {
+                  return MaterialApp.router(
+                    title: kAppName,
+                    theme: lightTheme,
+                    darkTheme: darkTheme,
+                    themeMode: ThemeMode.system,
+                    routerConfig: CustomRouterConfig.create(context),
+                    debugShowCheckedModeBanner: false,
+                  );
+                },
+              ),
             );
           },
         ),