From 9bc83ce7e3648e078f778b569a068c3d4aac5189 Mon Sep 17 00:00:00 2001 From: Carl Amine Date: Mon, 2 Mar 2026 12:22:13 +0200 Subject: [PATCH 1/2] production setup: RAG implementation, config, deps, lifetime preload, renamed streamlit folder, updated docs and requirements --- .devcontainer/devcontainer.json | 4 +- README.md | 4 +- .../app/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 163 bytes backend/app/__pycache__/main.cpython-313.pyc | Bin 0 -> 1269 bytes .../api/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 167 bytes .../app/api/__pycache__/deps.cpython-313.pyc | Bin 0 -> 903 bytes .../v1/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 170 bytes .../api/v1/__pycache__/router.cpython-313.pyc | Bin 0 -> 514 bytes .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 180 bytes .../__pycache__/health.cpython-313.pyc | Bin 0 -> 456 bytes .../__pycache__/recipes.cpython-313.pyc | Bin 0 -> 960 bytes .../core/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 168 bytes .../core/__pycache__/config.cpython-313.pyc | Bin 0 -> 1275 bytes .../core/__pycache__/lifespan.cpython-313.pyc | Bin 0 -> 1298 bytes .../core/__pycache__/logging.cpython-313.pyc | Bin 0 -> 827 bytes backend/app/core/config.py | 6 + backend/app/core/lifespan.py | 11 +- .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 171 bytes .../__pycache__/recipe.cpython-313.pyc | Bin 0 -> 1423 bytes .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 172 bytes .../__pycache__/rag_service.cpython-313.pyc | Bin 0 -> 6386 bytes .../recipe_service.cpython-313.pyc | Bin 0 -> 1170 bytes backend/app/services/rag_service.py | 111 +++++++++++++++--- config/.devcontainer/devcontainer.json | 4 +- models/faiss.index | 0 models/recipe.pkl | 0 requirements.txt | 10 ++ 27 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 backend/app/__pycache__/__init__.cpython-313.pyc create mode 100644 backend/app/__pycache__/main.cpython-313.pyc create mode 100644 backend/app/api/__pycache__/__init__.cpython-313.pyc create mode 100644 backend/app/api/__pycache__/deps.cpython-313.pyc create mode 100644 backend/app/api/v1/__pycache__/__init__.cpython-313.pyc create mode 100644 backend/app/api/v1/__pycache__/router.cpython-313.pyc create mode 100644 backend/app/api/v1/endpoints/__pycache__/__init__.cpython-313.pyc create mode 100644 backend/app/api/v1/endpoints/__pycache__/health.cpython-313.pyc create mode 100644 backend/app/api/v1/endpoints/__pycache__/recipes.cpython-313.pyc create mode 100644 backend/app/core/__pycache__/__init__.cpython-313.pyc create mode 100644 backend/app/core/__pycache__/config.cpython-313.pyc create mode 100644 backend/app/core/__pycache__/lifespan.cpython-313.pyc create mode 100644 backend/app/core/__pycache__/logging.cpython-313.pyc create mode 100644 backend/app/schemas/__pycache__/__init__.cpython-313.pyc create mode 100644 backend/app/schemas/__pycache__/recipe.cpython-313.pyc create mode 100644 backend/app/services/__pycache__/__init__.cpython-313.pyc create mode 100644 backend/app/services/__pycache__/rag_service.cpython-313.pyc create mode 100644 backend/app/services/__pycache__/recipe_service.cpython-313.pyc create mode 100644 models/faiss.index create mode 100644 models/recipe.pkl diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c8d2955..101597a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ "codespaces": { "openFiles": [ "README.md", - "app/app.py" + "streamlit_app/app.py" ] }, "vscode": { @@ -18,7 +18,7 @@ }, "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y HwoWvNH)Z*-t{DK(2{L-S#;tWSmBvw*la&~H7N=#xwK}>vnW?p7Ve7s&k@!#7VkP)y4hXL ztW&5IQG^gdNYG2AaH1C^5C_DmXD*!U1Sw|JoDhfJ9EA%v{%-8J;?R-i|M&m=dH>JM zpW{?20XQBuf3SWL0R9S$(TEMf@k0c6K?Rj8L!D5P53sN-)I};H7FB6ksz+!f#F6D_ zU8Zt~qsy^+oW>E$TH-Z9jh%&$B&ulSdC(%w$x))QR^ytiCgwz%(o)THWB&+gqNap< z&yPG~eTkf4RZ){=63)~d)`U!januNDaW$8 z9{-NwF04WG@zpQ{;L=d5Wd3~s9phC}CNBih!QTH3RT!N)2k#Lklz|C3xLO2oGf!fF zamZufi?2KfutuH~y(+A{44^(th@w@h}!cRj7)cB#cSssdD^CKIVJBC zo^E+MWoB!5vowRWK?-&i#ck-z=`XiG+y4C1-3$Bjx&B0Mr@NEvPhb3g`@8KQ)jy^y z`^v)JbY-`2|N5`j?|<-hZKwL}(pO90)OypEp0e;jTKIQQSzrp1{XAJcFa0Jzjq+Ax zwp1z27DXQME#Ee|yoD#QT!+dS_%02?ixN-iw(Z^=mBuqCW}BLZ>G+mzvr)vR>2J8k zskve52J+pK%h)^RtqOKg)T3s{`nXZdaJhr6hzm4$c>e6rDs7&2WU1{^Mx|g?L1?1w zbgXq64LqsPGwRQ#dvg(=Exb$~tFErL8r7Azxri6f)4E}_#>sdx z_ztaW&Dv6}q4KolwC%284mx$TuAh45NS0*bwm3)vkq#i+gKS^Q z{+Z3+n>;97?iDWYXJ7B1Ior>jxib@rfdV9V1W_UkCQi59nZIW}fb1U3(iyDeKNuq} AR{#J2 literal 0 HcmV?d00001 diff --git a/backend/app/api/__pycache__/__init__.cpython-313.pyc b/backend/app/api/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c768ddabe360bba192c9c0ef048acedd969c186 GIT binary patch literal 167 zcmey&%ge<81Wy8&W`gL)AOZ#$p^VQgK*m&tbOudEzm*I{OhDdekklHwoWvNH)Z*-t{DK(2{L-S#;tWSmBvw*la&~H7N=#xw0T5@##K&jmWtPOp>lIYq g;;_lhPbtkwwJTx;8Vjxx1A+{BhXb4#au?x$(nQYd?-Q~?j^dR(~ zDD>dfgMW!viw9w7PI~DrvUqo9k~E1A=FK9YM zo8fF$u#od}X8ugnOw4iv7&@No^~jkrVX~AYNf7#!SMzPxw+OrsJi=GSHt~X(ObKQH zNrn)5(^KT6^DOR=HhDTi2mF4MI1t@sw6z(U&kI#!&d!j9?<-r#RGOy3N?kXg)Cs~4 z`QXHE(qko%s;Cq92a+*05KTa6VWlV4nq@h)=HC7*r&WFNl$F~1yS3K7<+K{lB=Z%e z;w1KX_ksPA5}>x{!oaon2<<0PY&WAJ1hiMJEuc59*C%1ecH@|j!0wQkR^pFr)hCGq zuJ7-z&--XMk<;|bvos;s;IryWi7mP7dbpuxUt|dc-S<=ug4weiR ztc=++bDQwQCd^c>!Bjw;EG58-K%zU8cw(2NPHEh^)HPlZJE+)kh)xSvKRZ9JRZa@q z-?eS|ZPU!NTz43HNfZqz2nDR*ML;TE6m|oDV)@$i?zFOKgm@BmWnNl7doEO{_+M@I z;k&3a9FRv)=2h`e(R)0NRgCcnT_2$vBXo0wZjTM56vm2%jpOwc@hMej#CQecTgT7; JB0kBx`~!$h%q#!^ literal 0 HcmV?d00001 diff --git a/backend/app/api/v1/__pycache__/__init__.cpython-313.pyc b/backend/app/api/v1/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1bbd11b23d21244aa2a180a41bac59779e88f0e GIT binary patch literal 170 zcmey&%ge<81Wy8&W`gL)AOZ#$p^VQgK*m&tbOudEzm*I{OhDdekklHwoWvNH)Z*-t{DK(2{L-S#;tWSmBvw*la&~H7N=#xw0T5@#lo`gv$7kkcmc+;F j6;$5hu*uC&Da}c>D`Ewj46?Zx#Q4a}$jDg43}gWStyU~F literal 0 HcmV?d00001 diff --git a/backend/app/api/v1/__pycache__/router.cpython-313.pyc b/backend/app/api/v1/__pycache__/router.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ab89044da8f116a58f11c85fb7df580f098eb09 GIT binary patch literal 514 zcmey&%ge<81iisaGZh&b7#@Q-Fu)3Be69jArZNOG1T%Uwcrg|+Dlh~y1v7gyc`+BU zK-sK7Hk$%N45Jc*DO6h#dklM!Ae4hd#jpgk=rabhS~3CEV^_fz%m!1D&aTPv5~N#` z@fN3JfM-yCX-R6)OQ2gpB5Xxq?k%>A)Wn>U44?>0Nn(2OE%u_+^HK`(GxJJ{^&onRK;FE?3*p2=%@ahZf|_)T56XZk;Q|^7WAkO^ zCFhi;q(b>C89oEeWVq$-Y!wq)oLW>Ilbl$TlNjTYTAW>yUl8M$Us{w|oZ;w+#7as` z&IVcHwoWvNH)Z*-t{DK(2{L-S#;tWSmBvw*la&~H7N=#xw0T5@#lo`eVxdr)|c_qa$ r@$s2?nI-Y@dIgoYIBatBQ%ZAE?TT1|c7WVa3}Sp_W@Kb6Vg|AR)>ANU literal 0 HcmV?d00001 diff --git a/backend/app/api/v1/endpoints/__pycache__/health.cpython-313.pyc b/backend/app/api/v1/endpoints/__pycache__/health.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..16b359f1be8420c0d4ac637b2900091f3b146e22 GIT binary patch literal 456 zcmey&%ge<81Z#trX4V4f#~=<2Fhd!ieSnOq48aV+jNS}hj75wJ3^9yK45m=&BBmIo zAa*DRi3(;iWrT2pnWYe_S<+cFSzm%AG#PJkItF+K<(HPE7FDt9XQU?Plw>3$8Nt8= zZ}yG@1Qwu@#pjmXsFXV#?26$xsBc|1%IX+zNEIiU}=FEh>&l zPAtkvjB!aV&MwI>i1EuWEy^s;aP&lCB_$?jr{<-^Bo-6^ab`@JVGNL4ke``XQXB(u zj$T3KEj9?V2;>MhATH(w5)BLwctj?oT;P_zz#?750Tl7mWWB|nmI!oLL1q!i)wkG+ zz<#^MoSs?&)(v(iSaA_2NCAgUZhlH>PO4oIACL_bkWZs`WL h4{Quf9PLt#QXQU`8O1&_F*3@0W@ca#2eZJY0RX8uW@Z2Y literal 0 HcmV?d00001 diff --git a/backend/app/api/v1/endpoints/__pycache__/recipes.cpython-313.pyc b/backend/app/api/v1/endpoints/__pycache__/recipes.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6cbb193186e635b8fca546adb80eeb376edbf3cb GIT binary patch literal 960 zcmZ8fO=}ZD7@nEk>~7NTR@-8s(HcrBM5G_kQY%&{t!NVnk|lU>AgsyQrcJZ!%%)IJ z@#Mj){RJYlC;x(;yl(NJEJ_tmy`{t-;5*q(tPZ>HJnzT6-(Ay8AvkaCclDbDp2+#C{56+!y69WU}6{C z)1uqrb+Ns+c>z6js=F{1>UbXfb-KSmf%V*a(-$=E* z8uJ8~I~`8M^ANL9kF*<1Wm- z)nNDdRbb^b#bY2H1!H`QZhV*D(kXiQGdcP?-PSr9BF3xq%XDB)1#|1|bbvR`H8eF7 j%q)MtcKbY$zzbi^Oh;?VrWUH9a;?$zz znB>HwoWvNH)Z*-t{DK(2{L-S#;tWSmBvw*la&~H7N=#xwK}>RfQEE(ld}dx|NqoFs iLFFwDo80`A(wtPgB37WeAbX2JjE~HWjEqIhKo$U8wJW{= literal 0 HcmV?d00001 diff --git a/backend/app/core/__pycache__/config.cpython-313.pyc b/backend/app/core/__pycache__/config.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c3ac13fc6974e290d6438ba6753ca436f9defa2 GIT binary patch literal 1275 zcmYjROKTfP9G{g|58IL@+fu5S7*w0INTsrKa9f8GP^6W!wjv2lt(taWy4oG7i&wkq z%*r^zQ1w(pFS!JI=pmOv3;h877&gAd1Emc;8gr%T!Llq$JS$Lf|X!^=r5RI#va8WVqin^p7& zXI}sf5D5YjfD#Gb2b#=l2*4necLF2?LnN$4v?vUJ$ZfSbu8Y7Zoa4HAUp_yRFTmK2 zL>7E$d?sCl3Ew*A%acM5mq^^_mVIu8Ec#q(=97kZe4m6bf6uqdkYx96Zk0u+O}-Ib z4CMlLj#=31IxfL{B2mtwj%8yO!Y3Zlhw+*_AQt_$sAh`|l|_q<2C5aGDlBN?<6)A+ zDfRR&AK+=9xF!z~F>yy9*dALjF*S(Q_bk_8L8aDWvRbRua}rtPso7k$zKg0#OQ|v` zKfG6Rdp-SQiu*dzJ#1!OV)3Yy@7X+ymFi81>P>Z5t-;}P&o!}4H#@pTX~A+#e8Q5I zq6#5WYh~pSHHu$687@t%2sW%fF7%IVmTW2|wV^=N*suCV?EME@?vdL(>@~kC?|QWI z8QwO^C$Opv_rB~5Dm#un+$t067-u`KZDNu&T*uQbhxePkl|K7X_I8@xa#L+7s8X+% zl_pZlzk^zCiA51|^d3ft%_GE%8`y%5A@toqwZZEhAfB7w zOA-^JAoU32*1F@ko-RgAxwSY`k`%BkASECz;EsR{2bM;NADCwuNcTKqwFe#+o_D$K zhJS!5#zXCZVnQ`TC$_GYF+K9!zBaoEx2U#m9AU@Q^nPD6_zmJ9N-XrBkqD3FgQ$sq z&*4XKwQ~Qr<3A5CRw|?FKdFt;lQZK|-ndGnem*$gdVG;6jY?-LmvZSUz54Q#i*$aZ zp3zG=|2nhw(m4P0;3Cr+RiAHM%B{Cyu=e0wPCe)TTd(eIPRPycC|FxR&p&-t_~tq& zt+gdz@gI;vZHcT;E2`xpY^ndu)EzMfI(h#@{4mDzj!u>1fr?gGbY>?;H4e7nD?T5e tFo3!5c~B9AeQ>+;SW(i#{YQ%X;T0I literal 0 HcmV?d00001 diff --git a/backend/app/core/__pycache__/lifespan.cpython-313.pyc b/backend/app/core/__pycache__/lifespan.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..024cdd6eefa09bf1488684e6e6b8c3f20d7381cd GIT binary patch literal 1298 zcmah}&2Q936o2;EyJH77sptl}QOGo?(5=cAp+=OHqHg*@$yQs4l~XK5#yfW07~2|q zf#g!!OW{N>NV!)%^v_66JtiepBts++?IE{Ri>gZPfj8ORkfKVw!_52s_&tw1hYy>8 z^w-=^flUCON}*3E?@=ej>poauDGd7e;bEp@6 z;2H<3Frjo6qG+pD(bnEqEn0!EwTYgepuSQycBIs-D)|v7U5Z>cjxv5LyY57e&jqTB zpF3$*zjAp;W<&Wk^zZ_|E%)}5u^)FkP|0K++N+iGCH0_1?XHv?FnEw z-Q2#t-wA^`bm`KJjChGGr9TUUzA2aj!c2e8T5HJFC2e9<>E%*@*ISO6D@icC-aNT z<1_V3tY>72;9=~vc#EZN*X3#IwZm{}`Ly)u*_h`9A#c6Qw zdn;WYu@>6dUQ4a`X7t~_s#FxPeqNBD3KzK+L|&XzCkbjTo}@x`!^Rh@;y3@EqW-Iqo8l zS~h;8wi^p>hk?h_#EELjQcnAVIEuODQ%%3YV+U@-?^=0NJNig7zi)ioSRZ*T3 zRR2oZrnM)f(FdimJLEAPxM$uqe>l5x^^Ung#~#try@9&}>#u*XNzXr{pqu;gr+?FG ze?K$eP`LvNWwCb&p8aP3Hgu7iyppagkMMiySp*)oB48Y zJ)$I)a{Qm6gC~e#JaT&BdfaM<{GvFHqI|sR1w61rMR^FNZFpxJM*e`4kBy-%qw=d! iS^4mRaq4zqw*YE+R~=N|TPbz`uMR=QpHd-4F~M_%Hzx%E literal 0 HcmV?d00001 diff --git a/backend/app/core/__pycache__/logging.cpython-313.pyc b/backend/app/core/__pycache__/logging.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b78e9f3d3dbdaa3b0a8659c93854d006a2b71b25 GIT binary patch literal 827 zcmYjO&ubG=5PqBeku)*UMoU$M6rr^1p;9!2N>C*BmklXov6n2A*UjrTcC)+8dn?wH ze}uhx^-vHz3xfZGZ4Qywc=BRzLh$Z;Nw(X8o%!b5dEX2(!+buAgjKCiotlEsZ<)lF zntZ=3;D2dA@q=`@C+r4b$pjP zl8!AcfGWP>AK{KsE9Ox%ZhOffbRV_!I87>BM$9!cF-l@Hm*5zcCm_*iUW@-Fu`K5j zW+_+d3S;i77gODc^JXzdNi53+2zw}2wIY_}B2#U=Tbf_Q)M1@2G3ng=qDS5k&&QMO z5_Acr*cBzF%JoqVH}(VQV#YHE*l&9TsF~$yFK}J4Tb|p)wBytQf4}4Mv`ZLf3_ugz z`BZ&-eYd{4vs&kR6ov%2w%J(Q=6Q<&!QG9q8dvENef|SN`>_oG literal 0 HcmV?d00001 diff --git a/backend/app/core/config.py b/backend/app/core/config.py index ecbba0f..e700c0d 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -11,8 +11,14 @@ class Settings(BaseSettings): CORS_ORIGINS: str = Field(default="*", description="Comma-separated origins or *") + # local file paths for retrieval artifacts FAISS_INDEX_PATH: str = "models/faiss.index" RECIPES_PKL_PATH: str = "models/recipe.pkl" + # when artifacts are missing the service will try to download everything + # from this Google Drive folder ID via gdown + GDRIVE_FOLDER_ID: str = Field(default="10okoRXmRZDGtsF8e5cDxSLEzJWfuF9nl", + description="Drive folder containing faiss.index + recipe.pkl") + settings = Settings() \ No newline at end of file diff --git a/backend/app/core/lifespan.py b/backend/app/core/lifespan.py index e801a5c..45a2839 100644 --- a/backend/app/core/lifespan.py +++ b/backend/app/core/lifespan.py @@ -11,6 +11,15 @@ async def lifespan(app: FastAPI): configure_logging() logger.info("Starting NourishAI API") - # TODO: init shared resources (FAISS load, embeddings warmup, DB pool) + # initialize shared resources; failure to load RAG should not prevent boot + try: + from ..api.deps import get_rag_service + + # instantiate service to trigger artifact loading only; do not call `search` + _ = get_rag_service() + logger.info("RAG service initialized (artifacts may or may not be present)") + except Exception as exc: # pragma: no cover - artifacts may be absent + logger.exception("RAG init failed, continuing without it: %s", exc) + yield logger.info("Shutting down NourishAI API") \ No newline at end of file diff --git a/backend/app/schemas/__pycache__/__init__.cpython-313.pyc b/backend/app/schemas/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6545a8f891933952759a1d93b2a3b427ad7561a GIT binary patch literal 171 zcmey&%ge<81Wy8&W`gL)AOZ#$p^VQgK*m&tbOudEzm*I{OhDdekklHwoWvNH)Z*-t{DK(2{L-S#;tWSmBvw*la&~H7N=#xwK}>OSMrv+iaZG%CW?p7V le7s&knIMHLmVjxDMk?}K)~LB5gBnnYH2i??Ol&No>^DV zN;s!TImEtg4v~E15()kd!IHJ~B?nG>L+o;=s%H&0$w>1}^;g|BU0+pA=H_Y)pWlKP z$zKLzf6(IOac0J(i^g*nu)w&_+6EYH6U-K^&HGl{2D|Nm(_)v|6&6@aEU>xpkyV}r zSF;YX?kTpS*$T4WDc0*%{o0rg`lhOW7D@3<-Vv#CzDz{gLC*d;VW?bzQ z=V`7A^`7h3l>0!$NnecVfHISga*Cu#g{mi64@4&sSs_&;$z%b8m@<%?%8R2kheSx_ z$T$b_3-cSw3&Skh5n-rmVYrib1}WK%Fx(wP>CAIJ47&-)B26-p<>0N z_%H;z3!Ecx=~hlR&7%U&lf4jzk~YR+R1}ar927$0iMP-+q=G=Q7?BJCWp|2Q;k|=TkL_hx zJPCKE0VqLwChG{VHvsmg8CZe6W`e_AUI}!OxsALVR01!ku33QPMo&$bwGOpj!=FweNH zzbVn{7z_v~s%#)&x6L(&*ADoxy;x>L?oGd{y#8J%%5VsP($T6ax7h2`$m(BFSC##% z_0zG>3ah+3<&1s;H*hl*R^CMT(=rU>6WoS~HBL M!?nK|e)PTm0nqA9M*si- literal 0 HcmV?d00001 diff --git a/backend/app/services/__pycache__/__init__.cpython-313.pyc b/backend/app/services/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..34c3ee6ec720d3fcc6dd51010480d871010a4c43 GIT binary patch literal 172 zcmey&%ge<81Wy8&W`gL)AOZ#$p^VQgK*m&tbOudEzm*I{OhDdekkl>jWUH9a;?$zz znB>HwoWvNH)Z*-t{DK(2{L-S#;tWSmBvw*la&~H7N=#xwK@3n!S!Qx-aZG%CW?p7V le7s&koPgTB2lIrmWRhZKjrF$x>`9j(v(LDY9iM=8A4wvZYt#lHQ0~ zGP5h&(%lt7e;lN1ud>>tR?(tx`T_g?Il!THfC3SkCe2SVl8?NNiUw%@1ouZh`z}R; z0)1zfOVTTC3#68d^aU%nUSKeD!G>*|(Z-JLdK(tAdg{PVNa>5N3vTS@P#i^C&7rDk zC*rNcl<|!{2M~NSt%$b`TgqkL3>CD0Og_RvTBAqhMa>$LlB#B%k`y&aX-&K+tD>9~ zXRshE=^UOH@lu1FTfmZXO_2JqkVzV{jM!2Bc$9+7YoHdH8cd7AG;hIHo{lrTHO>rK zi6Spqv5oiete&$&&Y|a6-U+Q-aR+aUJL9f66L$|;7?4MPLpI*6w`$-UuqR^=dNpT6 z$nYY*E+xf}8=;9{fhjx3M<;o{cUDlu6o*9>OX77Q%PB^?K8~BYCUVJ~tf+V)spc>z z3KNP=yNOTHVmC`~|RXFh`EDCotmAfHlb3#gzGh7c0rM3=nWk* zQc~{^YA;|};RG&A2F|6q6zq+ZlLeTK;=CcL*Pt$$o6j$(Fw3h-GKa;hTv`&dsXnLX zOAC^sBqTW{-b~~L^_u3xVp7VBN+Lg()k~5ztd|)`p8OuIL-ZPY(U&!A(2HEQe2fr^ zk!{cePnm}+in0|+Jc2LtxTAjZRp}8$kp{DOH61QzIcOX4R?E#?wLXJ_mhjS-l=7kM zRja2`=@+uu-b<1!O8eD9Oqk<1Eu0*QN8wy4@uYyWLOdiYb80Rh4;zm6) zx0)l%*6%n=!CpP-?1sd>YeSBPB73;N9^PiT66-FqfdU(N$R2(KVvuGtj#)Ta*g1aJ;UidBF=5~#K^bDbzkf5A-u|GQ)7?L2$h z%AriHuI=WS)s%Odt3%pmPzQ?n&00GW02c_o$#gpke=Ciig)~j=X~Q8TTWeON-YAZ! zwYnnhM51-nnklDg%9I>*g=c`%SRKa`xw((YStbRFsH%YG%F>Z*s+w0u`ukG^()%*G zTqY~_0m2#u{pnmb1vkBaX)rQ6!Byabn+Ko;0mnE&RmJ%{5yqJ`VQ@r`G7cDG%Ppt? z)*AEjf+VV%Ye5kc$y{y@;C!hY;5rL14jfVG`ot(BT#vFeAVamzH_^xxCtQa>$jbuIIl)NHrmu;jW2NRIJOZ zBGtZ$P-ODTSYmv7Dl`^JOolYa*v+H}6V1t*4Hyf!Q?p8PI;SxiqP^xd=a4YgOtaq* zunhVI8B8vNA=hD4CZv=`FW{_3i#L-BVG96Oj>Ao)A-trJ7N~{7y=dJ7EfhPz*DGj; zb*{Sp+4bskwNi)uxt=N>K2bP) zqUaqec!z%d+*=cGPP`R@4-?$8O|r zdxWTlL#3A1@6o$7U&?3#SW%2Ti)ziHhc z>OpUN&xTH--}bmeL-cP?P$WL-9P6Opp<2cQ^g96t^7Vng1n~bwz#k!7wI=`o6RzD8 zOcJ~LNMdFnGHmC1}l(_D>&{negk zj;Fs!LJi2)p~%lzsRO3`YWlxc>R|)< zqre)H`=tl4C*aSN1LXVpYs$pB%$}eVXXdME)y}BwdNYt4xvhGr-^Ot zCJ=P`sp?p7*BqZAp|bBfB~{27C<#=RshXLC#(M*AiBi>7(x1DPbEuOKHA+oTVF%C< zXqKc72>AQ3s62DLG|wwe=* z@SxY{iOHeCu(}A7{lH&>NMm#@@RvyedHZSBTwVlnGt|(mWG(Qt*86^2@4Zy9?|h-}{8n4&mi1L-7e=NZ`d)bKL=8== z;g#^_p>rGl;^>va=#_2vS3dge5m*?Z`Jg%OegAgz(+@g!x`KC~`ROzFUjEtHTcKB{ zKA};|$&XB;JGnD(>Vf~Qz?*@M#p1x_!ocMppTA?h>wew69q8Ts%GXK(t{5071P1<_ zvFtkmtYvES+(xJv`f4Hc)ran{{rPd1PH`3(9t6AB+?#D@9wfJY&z0IbZdq?TzvujP zmx0>9eBy(_A2vVzDa?(KVr`Fe8%^83u~Iwf0V2vhlu20I`=iX*AbOi=89#;IJ`@-~ zLce`-^dNk^Gw2-err$Z!Hr`IZ+s;7#-6PKNVf(uScBp;#lyf3T|IX4j(M|uZn}Ixe zu!(q4W5BRW%spW>lh-o=q(|QcP#YATRY>0AX5 zfVvvGF?}x8hTudpVUz<4yqn0YL$LL1@hH__vFgDtsjFc<8$DKmoqDzou|kMNblL*c zQwHA!Nb_U;NVdf)AR7h05P4HX86!2@HlgQVMM&>)Gs^PJ2*6|tY@;X}tH6H!H8a;? z6f~(%ucEdH0k%CMT4QBh`T;^Npp>Ys8wKs*r2sJ@6vNe|pW$bFFqjx{ zik}5*;_u2eN${Nj7tAiGHAam9UMT|T28tgeUNmq=5fg)$IE%VdO0%b-T0JocW)zcB z7xN+}+p96UFDlqrgR(m45&$(YmiU3d?2wb-lF@929-2kYYj!|ajD5k^--vw>p;{3}710m^5|Xy=FoTAZKNsE|fY61R`C zkX6ABN`Y5RqoveMjRu0zD6A0R*G&&2GEjVs2-sJH;)b#jC|3Ro5wSWvtF9H-rhoAM z*fu*_a(P!rRz?c0wmUO-<9Fjb?nVQSzrOAMMyaWJdGbR~(QfTa2 zKELBVxZd?+wdn0Gc)Lsf{kNZAd;a?u)~F95JGC}d^!F6}JzM@Gx2TMQBy9kndw4U6z-k*8U^+0)W^o?&6 zj)y)WRmMm}KG@%OVj;uDG2SPqpir~HT_Gj9&bs#cJcghS2T21)UYO4>;&GC1HXWXc zT8F3M({yA+bxJqDjk~7L>nagLc1EA8#-1^j`z%gF8N4@;at0zW7b)rkk!8&uuxr{EF=p&kRNF|1U&hUHabuxRW>% literal 0 HcmV?d00001 diff --git a/backend/app/services/__pycache__/recipe_service.cpython-313.pyc b/backend/app/services/__pycache__/recipe_service.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d2ddf88a64fe19e548e103b5e5b18f9affa3796 GIT binary patch literal 1170 zcmY*Y&1(};5TCbSNt3pzty)nJ!4GtcR1Z~ANA7m0Sn3CkU*A2go>}!5Ny|@T@Uv z&Ws6~vuxltwhV{kNkUFVcRu6Ap_hw{>(hGd7``)ckE+vhoS822` zlC>Lm_n~<|SPJ6>&u=z8O3|ZB4^vh|#~y92sJswS93?`iqX)Vl?e1%eq*l-2`_R#s z`8GjN_LEkKE6S$oVo5m|CM}EuJy%!q1kyR55?F^BnC^P^t<(E6I{R~joLw_wbH*Hk zSxXuhK^jAFryn4;9%<%N;l?GHv2L*|?16Egs4PO6;$rIz=?pD`YgSwXhbSHoliH$< zluqTy(8r*L`ZU)@IvW0m=_`At7)tFeYrzsB)t@Daj6NtwesYEUaZIremwe$ny*sTZ(oFnf4o@(<9m>lpUP z=FMH8<)fJ{+t&8%r`|oilO3ebsJJ=hBrDA*uBMF;r9mxR_T^~CY1)`7(ihWxq1g;+ p{ewNJ{CGyg1fHXl^=B@Rk=ivGV?SZ|C!D6=p5w5Q-#{vD;vb9N5JCU| literal 0 HcmV?d00001 diff --git a/backend/app/services/rag_service.py b/backend/app/services/rag_service.py index 8d0ffa5..e985c32 100644 --- a/backend/app/services/rag_service.py +++ b/backend/app/services/rag_service.py @@ -1,28 +1,111 @@ import logging -from typing import List +import os +import pickle +from typing import Any, Dict, List + +import faiss +import gdown +import numpy as np +from sentence_transformers import SentenceTransformer logger = logging.getLogger("nourishai.rag") + +# we'll construct the download URL from settings when needed + + class RagService: """ - Replace search() with your real FAISS + embeddings retrieval. - Keep this free of FastAPI imports: pure business logic. + FAISS-based retrieval service. The constructor eagerly ensures that the + artifacts exist (downloading from Drive if necessary) and then loads them + into memory. Search is pure business logic and returns a list of dictionaries + with a computed `score` field. """ def __init__(self, faiss_index_path: str, recipes_pkl_path: str): self.faiss_index_path = faiss_index_path self.recipes_pkl_path = recipes_pkl_path + # transformer model is expensive to download; load lazily in search + self._model: SentenceTransformer | None = None + self.index: faiss.Index | None = None + self.recipes: List[Any] = [] + + # ensure the FAISS/recipe artifacts are on disk (download if necessary) + self._ensure_artifacts() + self._load_artifacts() + + def _ensure_artifacts(self) -> None: + os.makedirs(os.path.dirname(self.faiss_index_path), exist_ok=True) - def search(self, query: str, top_k: int) -> List[dict]: + if not os.path.exists(self.faiss_index_path) or not os.path.exists(self.recipes_pkl_path): + # fetch the ID from settings if available + try: + from ..core.config import settings + folder_id = settings.GDRIVE_FOLDER_ID + url = f"https://drive.google.com/drive/folders/{folder_id}" + except Exception: # settings might not be initialised yet + url = None + + if url: + logger.info("RAG artifacts missing, attempting download from %s", url) + try: + gdown.download_folder(url, output=os.path.dirname(self.faiss_index_path), quiet=False, use_cookies=False) + except Exception as exc: # pragma: no cover - network may be unavailable + logger.warning("failed to download RAG artifacts: %s", exc) + else: + logger.warning("no drive URL available, skipping download attempt") + + def _load_artifacts(self) -> None: + if os.path.exists(self.faiss_index_path): + try: + self.index = faiss.read_index(self.faiss_index_path) + logger.info("loaded faiss index from %s", self.faiss_index_path) + except Exception as exc: + logger.error("error loading faiss index: %s", exc) + self.index = None + else: + logger.warning("faiss index path %s does not exist", self.faiss_index_path) + + if os.path.exists(self.recipes_pkl_path): + try: + with open(self.recipes_pkl_path, "rb") as f: + self.recipes = pickle.load(f) + logger.info("loaded %s recipes from %s", + len(self.recipes) if hasattr(self.recipes, "__len__") else "?", + self.recipes_pkl_path) + except Exception as exc: + logger.error("error loading recipes pickle: %s", exc) + self.recipes = [] + else: + logger.warning("recipes pickle path %s does not exist", self.recipes_pkl_path) + + def search(self, query: str, top_k: int) -> List[Dict[str, Any]]: logger.info("RAG search query=%r top_k=%s", query, top_k) - return [ - { - "id": "demo-1", - "title": "Demo Recipe", - "ingredients": ["ingredient 1", "ingredient 2"], - "instructions": ["step 1", "step 2"], - "calories": 420, - "score": 0.95, - } - ][:top_k] \ No newline at end of file + if self.index is None or not self.recipes: + logger.warning("search called but artifacts are not loaded, returning empty list") + return [] + + # load the encoder only when we actually need it + if self._model is None: + try: + self._model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") + except Exception as exc: # network or model error + logger.error("failed to load sentence-transformer model: %s", exc) + return [] + + emb = self._model.encode([query]) + # faiss expects float32 + D, I = self.index.search(np.asarray(emb, dtype="float32"), top_k) + results: List[Dict[str, Any]] = [] + for dist, idx in zip(D[0], I[0]): + if idx < 0 or idx >= len(self.recipes): + continue + rec = self.recipes[idx] + if isinstance(rec, dict): + r = rec.copy() + else: + r = rec.__dict__.copy() + r["score"] = float(dist) + results.append(r) + return results diff --git a/config/.devcontainer/devcontainer.json b/config/.devcontainer/devcontainer.json index 821e544..d09ba9b 100644 --- a/config/.devcontainer/devcontainer.json +++ b/config/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ "codespaces": { "openFiles": [ "README.md", - "app/app.py" + "streamlit_app/app.py" ] }, "vscode": { @@ -18,7 +18,7 @@ }, "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y Date: Sun, 15 Mar 2026 19:32:22 +0200 Subject: [PATCH 2/2] feat: productionize retrieval, grounding, evals, and ops --- .github/workflows/ci.yml | 21 + .gitignore | 16 + README.md | 325 ++++++++++--- backend/.env.example | 24 + backend/app/__pycache__/main.cpython-313.pyc | Bin 1269 -> 0 bytes .../app/api/__pycache__/deps.cpython-313.pyc | Bin 903 -> 0 bytes backend/app/api/deps.py | 20 +- .../api/v1/__pycache__/router.cpython-313.pyc | Bin 514 -> 0 bytes .../__pycache__/health.cpython-313.pyc | Bin 456 -> 0 bytes .../__pycache__/recipes.cpython-313.pyc | Bin 960 -> 0 bytes .../api/v1/endpoints/grounded_recommend.py | 18 + backend/app/api/v1/endpoints/health.py | 50 +- backend/app/api/v1/endpoints/meal_plans.py | 18 + backend/app/api/v1/endpoints/nutrition.py | 18 + backend/app/api/v1/endpoints/recipes.py | 14 +- backend/app/api/v1/endpoints/recommend.py | 24 + backend/app/api/v1/endpoints/retrieval.py | 15 + backend/app/api/v1/router.py | 11 +- .../core/__pycache__/config.cpython-313.pyc | Bin 1275 -> 0 bytes .../core/__pycache__/lifespan.cpython-313.pyc | Bin 1298 -> 0 bytes .../core/__pycache__/logging.cpython-313.pyc | Bin 827 -> 0 bytes backend/app/core/config.py | 46 +- backend/app/core/lifespan.py | 28 +- backend/app/core/logging.py | 78 ++- backend/app/core/observability.py | 78 +++ backend/app/core/validation.py | 39 ++ backend/app/ingestion/__init__.py | 1 + backend/app/ingestion/chunking.py | 100 ++++ backend/app/ingestion/cli.py | 22 + backend/app/ingestion/loaders.py | 48 ++ backend/app/ingestion/models.py | 13 + backend/app/ingestion/pipeline.py | 298 ++++++++++++ backend/app/main.py | 6 +- backend/app/repositories/lexical_index.py | 111 +++++ backend/app/repositories/vector_store.py | 106 +++++ .../__pycache__/recipe.cpython-313.pyc | Bin 1423 -> 0 bytes backend/app/schemas/ai.py | 112 +++++ backend/app/schemas/recipe.py | 56 ++- backend/app/schemas/retrieval.py | 76 +++ .../__pycache__/rag_service.cpython-313.pyc | Bin 6386 -> 0 bytes .../recipe_service.cpython-313.pyc | Bin 1170 -> 0 bytes backend/app/services/embedding_service.py | 23 + backend/app/services/filters.py | 118 +++++ backend/app/services/grounded_ai_service.py | 450 ++++++++++++++++++ backend/app/services/ingredient_normalizer.py | 67 +++ backend/app/services/llm_service.py | 107 +++++ backend/app/services/model_store.py | 26 +- backend/app/services/rag_service.py | 420 +++++++++++++--- backend/app/services/recipe_service.py | 71 ++- backend/app/services/retrieval_service.py | 188 ++++++++ backend/requirements.txt | 24 +- backend/tests/conftest.py | 6 + backend/tests/test_endpoints.py | 67 +++ backend/tests/test_grounded_fallback.py | 20 + backend/tests/test_grounded_llm_failure.py | 28 ++ backend/tests/test_health_endpoints.py | 76 +++ backend/tests/test_ingredient_normalizer.py | 15 + backend/tests/test_invalid_filters.py | 16 + backend/tests/test_llm_service.py | 50 ++ backend/tests/test_rag_filters.py | 24 + backend/tests/test_rag_missing_index.py | 10 + backend/tests/test_startup_validation.py | 50 ++ data/README.md | 34 ++ data/ingredients/sample_ingredients.json | 47 ++ data/nutrition/sample_nutrition.json | 23 + data/recipes/sample_recipes.json | 93 ++++ data/sample_recipes.json | 172 +++++++ docker-compose.yml | 2 +- eval/__init__.py | 1 + eval/datasets/grounded_generation_eval.json | 69 +++ eval/datasets/retrieval_eval.json | 46 ++ eval/metrics/generation_metrics.py | 107 +++++ eval/metrics/retrieval_metrics.py | 44 ++ eval/runners/run_generation_eval.py | 345 ++++++++++++++ eval/runners/run_retrieval_eval.py | 200 ++++++++ models/faiss.index | 0 models/recipe.pkl | 0 requirements.txt | 2 + scripts/build_rag_index.py | 136 ++++++ 79 files changed, 4778 insertions(+), 191 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore delete mode 100644 backend/app/__pycache__/main.cpython-313.pyc delete mode 100644 backend/app/api/__pycache__/deps.cpython-313.pyc delete mode 100644 backend/app/api/v1/__pycache__/router.cpython-313.pyc delete mode 100644 backend/app/api/v1/endpoints/__pycache__/health.cpython-313.pyc delete mode 100644 backend/app/api/v1/endpoints/__pycache__/recipes.cpython-313.pyc create mode 100644 backend/app/api/v1/endpoints/grounded_recommend.py create mode 100644 backend/app/api/v1/endpoints/meal_plans.py create mode 100644 backend/app/api/v1/endpoints/nutrition.py create mode 100644 backend/app/api/v1/endpoints/recommend.py create mode 100644 backend/app/api/v1/endpoints/retrieval.py delete mode 100644 backend/app/core/__pycache__/config.cpython-313.pyc delete mode 100644 backend/app/core/__pycache__/lifespan.cpython-313.pyc delete mode 100644 backend/app/core/__pycache__/logging.cpython-313.pyc create mode 100644 backend/app/core/observability.py create mode 100644 backend/app/core/validation.py create mode 100644 backend/app/ingestion/__init__.py create mode 100644 backend/app/ingestion/chunking.py create mode 100644 backend/app/ingestion/cli.py create mode 100644 backend/app/ingestion/loaders.py create mode 100644 backend/app/ingestion/models.py create mode 100644 backend/app/ingestion/pipeline.py create mode 100644 backend/app/repositories/lexical_index.py create mode 100644 backend/app/repositories/vector_store.py delete mode 100644 backend/app/schemas/__pycache__/recipe.cpython-313.pyc create mode 100644 backend/app/schemas/ai.py create mode 100644 backend/app/schemas/retrieval.py delete mode 100644 backend/app/services/__pycache__/rag_service.cpython-313.pyc delete mode 100644 backend/app/services/__pycache__/recipe_service.cpython-313.pyc create mode 100644 backend/app/services/embedding_service.py create mode 100644 backend/app/services/filters.py create mode 100644 backend/app/services/grounded_ai_service.py create mode 100644 backend/app/services/ingredient_normalizer.py create mode 100644 backend/app/services/llm_service.py create mode 100644 backend/app/services/retrieval_service.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_endpoints.py create mode 100644 backend/tests/test_grounded_fallback.py create mode 100644 backend/tests/test_grounded_llm_failure.py create mode 100644 backend/tests/test_health_endpoints.py create mode 100644 backend/tests/test_ingredient_normalizer.py create mode 100644 backend/tests/test_invalid_filters.py create mode 100644 backend/tests/test_llm_service.py create mode 100644 backend/tests/test_rag_filters.py create mode 100644 backend/tests/test_rag_missing_index.py create mode 100644 backend/tests/test_startup_validation.py create mode 100644 data/README.md create mode 100644 data/ingredients/sample_ingredients.json create mode 100644 data/nutrition/sample_nutrition.json create mode 100644 data/recipes/sample_recipes.json create mode 100644 data/sample_recipes.json create mode 100644 eval/__init__.py create mode 100644 eval/datasets/grounded_generation_eval.json create mode 100644 eval/datasets/retrieval_eval.json create mode 100644 eval/metrics/generation_metrics.py create mode 100644 eval/metrics/retrieval_metrics.py create mode 100644 eval/runners/run_generation_eval.py create mode 100644 eval/runners/run_retrieval_eval.py delete mode 100644 models/faiss.index delete mode 100644 models/recipe.pkl create mode 100644 scripts/build_rag_index.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4344aca --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: CI + +on: + push: + branches: ["main"] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install backend dependencies + run: python -m pip install --upgrade pip && python -m pip install -r backend/requirements.txt + - name: Run tests + run: python -m pytest backend/tests + - name: Import eval runners + run: python -c "import eval.runners.run_retrieval_eval, eval.runners.run_generation_eval" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..64384ec --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +__pycache__/ +*.pyc +.env +artifacts/ +backend/artifacts/ +.pytest_cache/ +pytest-cache-files-*/ +*.log +uvicorn.out +uvicorn.err +.uvicorn.pid +.uvicorn.job +models/ +!models/.gitkeep +eval/outputs/ +!eval/outputs/.gitkeep diff --git a/README.md b/README.md index 7fd8c08..4291499 100644 --- a/README.md +++ b/README.md @@ -1,66 +1,271 @@ -# 🌿 NourishAI +# NourishAI -AI-powered recipe generation — identify dishes from ingredients or photos using ML models and a Streamlit interface. +Production-style nutrition and recipe intelligence system with FAISS retrieval, metadata-aware filtering, grounded OpenAI outputs, and reproducible evaluation. -## 📁 Project Structure +## RAG Setup +1. Build the index: + `python scripts/build_rag_index.py --recipes data/sample_recipes.json --out-dir models/` +2. The index stores FAISS embeddings for each recipe title + ingredient list using `all-MiniLM-L6-v2`, plus a `recipe.pkl` payload. +3. Query it via `POST /api/v1/recipes/search` or `POST /api/v1/recipes/recommend`. + +## OpenAI Setup (Phase 2) +1. Set `OPENAI_API_KEY` in `backend/.env` (see `backend/.env.example`). +2. Optionally set `OPENAI_MODEL`, `OPENAI_TEMPERATURE`, and `OPENAI_MAX_TOKENS`. +3. Optional: `OPENAI_TIMEOUT_SEC` controls request timeout, and `STRICT_STARTUP=true` fails boot if artifacts are missing. +4. Use grounded endpoints: + - `POST /api/v1/recipes/recommend/grounded` + - `POST /api/v1/meal-plans/generate` + - `POST /api/v1/nutrition/qa` +Outputs are schema-validated and grounded in retrieved recipes. + +## Current Retrieval Path +1. Normalize ingredients and query text. +2. Embed with `sentence-transformers/all-MiniLM-L6-v2`. +3. FAISS retrieves candidates by cosine similarity. +4. Apply metadata filters (cuisine, meal type, dietary tags, macros, time). +5. Optional lightweight reranking by ingredient overlap. + +## Architecture (Current) ``` NourishAI/ -├── app/ -│ └── app.py # Main Streamlit frontend application -├── backend/ -│ └── main.py # FastAPI backend for RAG-based recipe search -├── notebooks/ -│ ├── 490EDA.ipynb # Exploratory Data Analysis -│ ├── 490Image.ipynb # Image model training -│ ├── build_index.ipynb # FAISS index construction -│ └── demo_RAG.ipynb # RAG pipeline demo -├── templates/ -│ └── index.html # HTML frontend template -├── assets/ -│ ├── Animation.json # Lottie animation 1 -│ └── Animation1.json # Lottie animation 2 -├── config/ -│ └── .streamlit/config.toml # Streamlit configuration -├── .devcontainer/ -│ └── devcontainer.json # GitHub Codespaces config -├── .github/ -│ └── workflows/ -│ └── docker-image.yml # Docker CI/CD workflow -├── DockerFile # Docker build instructions -└── requirements.txt # Python dependencies -``` - -## 🚀 Features - -- **Image-Based Recipe Detection**: Upload a food photo → get the dish name + full recipe -- **Ingredient-Based Recipe Search**: Enter ingredients → retrieve matching recipes via FAISS -- **RAG Pipeline**: Semantic search using `all-MiniLM-L6-v2` + FAISS index -- **Lottie Animations**: Smooth animated feedback during processing - -## ⚙️ Setup & Running - -```bash -# Install dependencies -pip install -r requirements.txt - -# Run the Streamlit app -streamlit run streamlit_app/app.py - -# Run the FastAPI backend (optional) -uvicorn backend.app.main:app --host 0.0.0.0 --port 8000 -``` - -## 🔗 Model Downloads - -| Asset | Link | -|---|---| -| FAISS Index + recipe.pkl | [Google Drive](https://drive.google.com/drive/folders/10okoRXmRZDGtsF8e5cDxSLEzJWfuF9nl?usp=sharing) | -| Image Model (490Image.pkl) | [Google Drive](https://drive.google.com/file/d/1kyATzxBuLP5nWScPpwT9KrkkIuxY22vX/view?usp=sharing) | - -## 🐳 Docker - -```bash -docker pull carlamine/nourishai-app:latest -docker run carlamine/nourishai-app:latest + backend/ + app/ + api/ + core/ + repositories/ + schemas/ + services/ + requirements.txt + data/ + eval/ + models/ + scripts/ + docker-compose.yml +``` + +### Retrieval Flow +1. User query hits `/api/v1/recipes/search` (or `/api/v1/recipes/recommend`). +2. Query is embedded with `sentence-transformers/all-MiniLM-L6-v2`. +3. FAISS retrieves candidate recipes with cosine similarity. +4. Optional metadata filters and lightweight reranking refine results. + +### Grounded Recommendation Flow +1. Retrieve recipes with FAISS + filters. +2. Provide retrieved recipe context to the LLM. +3. Produce structured recommendations with source IDs/titles. + +### Index Build Flow +1. Provide recipe JSON (see `data/sample_recipes.json`). +2. Run `python scripts/build_rag_index.py --recipes data/sample_recipes.json --out-dir models/`. +3. FAISS index + `recipe.pkl` are saved under `models/` for retrieval. + +## Setup + +### Local (Docker Compose) +``` +docker compose up --build +``` +This runs: +- FastAPI API on `http://localhost:8000` +- Streamlit on `http://localhost:8501` + +## Verification (2026-03-15, Windows PowerShell) +Commands executed successfully: +``` +python -m pip install -r backend\requirements.txt +``` +``` +$env:HF_HUB_OFFLINE='1'; $env:TRANSFORMERS_OFFLINE='1'; python scripts\build_rag_index.py --recipes data\sample_recipes.json --out-dir models\ +``` +``` +$env:HF_HUB_OFFLINE='1'; $env:TRANSFORMERS_OFFLINE='1'; @' +import json +import threading +import time +import httpx +import uvicorn +from backend.app.main import app +HOST = "127.0.0.1" +PORT = 8005 +BASE = f"http://{HOST}:{PORT}/api/v1" +config = uvicorn.Config(app, host=HOST, port=PORT, log_level="warning") +server = uvicorn.Server(config) +thread = threading.Thread(target=server.run, daemon=True) +thread.start() +for _ in range(30): + try: + httpx.get(f"{BASE}/health", timeout=1.0) + break + except Exception: + time.sleep(0.2) +client = httpx.Client(timeout=10.0) +print(client.get(f"{BASE}/health").json()) +print(client.get(f"{BASE}/ready").json()) +print(client.post(f"{BASE}/recipes/search", json={ + "query": "high protein dinner with chicken and quinoa", + "top_k": 3, + "filters": {"meal_type": ["dinner"], "dietary_tags": ["high_protein"]}, + "include_diagnostics": True +}).json()) +print(client.post(f"{BASE}/recipes/recommend/grounded", json={ + "ingredients": ["chicken", "garlic", "lemon"], + "dietary_notes": "low carb", + "top_k": 3, + "include_diagnostics": True +}).json()) +print(client.post(f"{BASE}/meal-plans/generate", json={ + "dietary_profile": "high protein", + "calorie_target": 1800, + "days": 2, + "meals_per_day": 3 +}).json()) +print(client.post(f"{BASE}/nutrition/qa", json={ + "question": "Is quinoa a good source of protein?", + "top_k": 3 +}).json()) +client.close() +server.should_exit = True +thread.join(timeout=5) +'@ | python - +``` +``` +$env:HF_HUB_OFFLINE='1'; $env:TRANSFORMERS_OFFLINE='1'; python -m eval.runners.run_retrieval_eval +``` +``` +$env:HF_HUB_OFFLINE='1'; $env:TRANSFORMERS_OFFLINE='1'; python -m eval.runners.run_generation_eval +``` +``` +python -m pytest backend\tests ``` + +### Legacy Ingestion (Optional) +The repo includes an older Qdrant-based ingestion prototype under `backend/app/ingestion`, but it is not used by the +current FAISS retrieval path. You can ignore it for the core workflow. + +## Example Output + +Request: +```json +{ + "query": "high protein dinner with chicken and quinoa", + "top_k": 3, + "filters": { + "meal_type": ["dinner"], + "dietary_tags": ["high_protein"] + }, + "include_diagnostics": true +} +``` + +Response (truncated, verified 2026-03-15): +```json +{ + "results": [ + { + "id": "rec-002", + "title": "Lemon Garlic Chicken", + "score": 0.28875704407691954, + "dietary_tags": ["high_protein", "gluten_free", "low_carb"] + } + ], + "diagnostics": { + "candidate_k": 15, + "reranked": true, + "top_ids": ["rec-002", "rec-008", "rec-006"] + } +} +``` + +### Grounded Recommendation Example +Request: +```json +{ + "ingredients": ["chicken", "garlic", "lemon"], + "dietary_notes": "low carb", + "top_k": 3, + "include_diagnostics": true +} +``` + +Response (truncated, verified without `OPENAI_API_KEY`): +```json +{ + "recommended_recipes": [], + "warnings": [ + "LLM generation failed; no grounded recommendations were returned." + ], + "source_recipes": [{ "id": "rec-002", "title": "Lemon Garlic Chicken" }] +} +``` + +### Meal Plan Example +Request: +```json +{ + "dietary_profile": "high protein", + "calorie_target": 1800, + "days": 2, + "meals_per_day": 3 +} +``` + +Response (truncated, verified without `OPENAI_API_KEY`): +```json +{ + "days": [], + "warnings": [ + "LLM generation failed; no meal plan was returned." + ], + "source_recipes": [{ "id": "rec-002", "title": "Lemon Garlic Chicken" }] +} +``` + +## Evaluation Workflow (Phase 3) +1. Retrieval eval: + `python -m eval.runners.run_retrieval_eval` +2. Grounded generation eval: + `python -m eval.runners.run_generation_eval` +Metrics include hit@k (any expected ID retrieved), recall@k (fraction of expected IDs retrieved), filter/exclusion violations, +source-ID grounding checks, and fallback correctness. Generation eval runs a deterministic offline LLM by default; pass +`--mode openai` to call the live API. +Artifacts are saved to `eval/outputs/` as JSON and CSV (run metadata, summary metrics, per-case details). + +Verified evaluation results (2026-03-15): +- Retrieval run: `eval/outputs/retrieval_eval_20260315_172249.json` + Summary: baseline hit_rate=0.80, recall_avg=0.92, filter_violations=14, excluded_violations=8; filtered/reranked recall_avg=1.00 with zero violations. +- Generation run: `eval/outputs/generation_eval_20260315_172302.json` + Summary: fallback_correct_rate=1.00, warning_correct_rate=1.00, source_id_violations=0, filter_violations=0. + +## Observability & Health +- Structured JSON logs include request IDs, retrieval filters, rerank flags, top IDs, and latency per stage. +- Request ID is returned in `X-Request-ID`. +- Health endpoints: + - `GET /api/v1/health` (basic liveness) + - `GET /api/v1/ready` (index + OpenAI readiness) + - `GET /api/v1/status` (alias of ready) +Example log (truncated): +```json +{"event":"rag_search","query":"chicken garlic lemon","strategy":"reranked","result_ids":["rec-002"],"timings_ms":{"total":12.4}} +``` + +## Troubleshooting +- `503 RAG index not available`: run `python scripts/build_rag_index.py --recipes data/sample_recipes.json --out-dir models/`. +- LLM endpoints warn when `OPENAI_API_KEY` is missing; retrieval still works. +- If `/ready` reports artifact mismatch, rebuild the FAISS index. +- In restricted network environments, set `HF_HUB_OFFLINE=1` and `TRANSFORMERS_OFFLINE=1` to use cached model files. +- `python -m pytest backend/tests` may warn if pytest cannot write cache on Windows; tests still run. + +## Resume Evidence +Implemented now: +- FAISS retrieval with metadata filters and lightweight reranking +- Structured LLM outputs grounded in retrieved recipes +- Reproducible eval scripts with JSON/CSV artifacts for retrieval + grounding +- Structured logging, request IDs, and health/ready endpoints + +Planned next: +- Background ingestion jobs with status tracking +- Expanded test coverage and observability + +## Notes +- The current recipe endpoint (`/api/v1/recipes/search`) deduplicates results at the recipe level. diff --git a/backend/.env.example b/backend/.env.example index cbc03c7..1ecbfcb 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,7 +1,31 @@ APP_NAME=NourishAI APP_VERSION=1.0.0 CORS_ORIGINS=["http://localhost:5173","http://localhost:3000"] +LOG_LEVEL=INFO +STRICT_STARTUP=false + +DATA_DIR=data +ARTIFACTS_DIR=artifacts +EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2 +EMBEDDING_DIM=384 +RETRIEVAL_CANDIDATES=50 +HYBRID_ALPHA=0.65 + MODELS_DIR=models GDRIVE_RAW_RECIPES_ID=1lE6zl-9dJKUnNu6CDuBvHPpGodKrGnoX IMAGE_MODEL_URL=https://github.com/CarlAmine/EECE-490/releases/download/v1.0/490Image.pkl IMAGE_MODEL_EXPECTED_SIZE_MB=687 +FAISS_INDEX_PATH=models/faiss.index +RECIPES_PKL_PATH=models/recipe.pkl +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o-mini +OPENAI_TEMPERATURE=0.2 +OPENAI_MAX_TOKENS=800 +OPENAI_TIMEOUT_SEC=20 + +# Legacy Qdrant ingestion (optional, not used by FAISS retrieval) +QDRANT_URL=http://localhost:6333 +QDRANT_COLLECTION=nourishai_chunks +LEXICAL_INDEX_PATH=artifacts/lexical_index.pkl +CHUNK_STORE_PATH=artifacts/chunks.jsonl +INGESTION_MANIFEST_PATH=artifacts/ingestion_manifest.json diff --git a/backend/app/__pycache__/main.cpython-313.pyc b/backend/app/__pycache__/main.cpython-313.pyc deleted file mode 100644 index c29d9c1c8fcb54f965979ce81548124c0a9c035c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1269 zcmaJ=L2MgE6#cWi_S#@!#7VkP)y4hXL ztW&5IQG^gdNYG2AaH1C^5C_DmXD*!U1Sw|JoDhfJ9EA%v{%-8J;?R-i|M&m=dH>JM zpW{?20XQBuf3SWL0R9S$(TEMf@k0c6K?Rj8L!D5P53sN-)I};H7FB6ksz+!f#F6D_ zU8Zt~qsy^+oW>E$TH-Z9jh%&$B&ulSdC(%w$x))QR^ytiCgwz%(o)THWB&+gqNap< z&yPG~eTkf4RZ){=63)~d)`U!januNDaW$8 z9{-NwF04WG@zpQ{;L=d5Wd3~s9phC}CNBih!QTH3RT!N)2k#Lklz|C3xLO2oGf!fF zamZufi?2KfutuH~y(+A{44^(th@w@h}!cRj7)cB#cSssdD^CKIVJBC zo^E+MWoB!5vowRWK?-&i#ck-z=`XiG+y4C1-3$Bjx&B0Mr@NEvPhb3g`@8KQ)jy^y z`^v)JbY-`2|N5`j?|<-hZKwL}(pO90)OypEp0e;jTKIQQSzrp1{XAJcFa0Jzjq+Ax zwp1z27DXQME#Ee|yoD#QT!+dS_%02?ixN-iw(Z^=mBuqCW}BLZ>G+mzvr)vR>2J8k zskve52J+pK%h)^RtqOKg)T3s{`nXZdaJhr6hzm4$c>e6rDs7&2WU1{^Mx|g?L1?1w zbgXq64LqsPGwRQ#dvg(=Exb$~tFErL8r7Azxri6f)4E}_#>sdx z_ztaW&Dv6}q4KolwC%284mx$TuAh45NS0*bwm3)vkq#i+gKS^Q z{+Z3+n>;97?iDWYXJ7B1Ior>jxib@rfdV9V1W_UkCQi59nZIW}fb1U3(iyDeKNuq} AR{#J2 diff --git a/backend/app/api/__pycache__/deps.cpython-313.pyc b/backend/app/api/__pycache__/deps.cpython-313.pyc deleted file mode 100644 index 25143e264c86ce0588528e161065b626d6610d4b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 903 zcmZ8f&ubGw6rR~%`DNND6-_~GrI$UJN>xx1A+{BhXb4#au?x$(nQYd?-Q~?j^dR(~ zDD>dfgMW!viw9w7PI~DrvUqo9k~E1A=FK9YM zo8fF$u#od}X8ugnOw4iv7&@No^~jkrVX~AYNf7#!SMzPxw+OrsJi=GSHt~X(ObKQH zNrn)5(^KT6^DOR=HhDTi2mF4MI1t@sw6z(U&kI#!&d!j9?<-r#RGOy3N?kXg)Cs~4 z`QXHE(qko%s;Cq92a+*05KTa6VWlV4nq@h)=HC7*r&WFNl$F~1yS3K7<+K{lB=Z%e z;w1KX_ksPA5}>x{!oaon2<<0PY&WAJ1hiMJEuc59*C%1ecH@|j!0wQkR^pFr)hCGq zuJ7-z&--XMk<;|bvos;s;IryWi7mP7dbpuxUt|dc-S<=ug4weiR ztc=++bDQwQCd^c>!Bjw;EG58-K%zU8cw(2NPHEh^)HPlZJE+)kh)xSvKRZ9JRZa@q z-?eS|ZPU!NTz43HNfZqz2nDR*ML;TE6m|oDV)@$i?zFOKgm@BmWnNl7doEO{_+M@I z;k&3a9FRv)=2h`e(R)0NRgCcnT_2$vBXo0wZjTM56vm2%jpOwc@hMej#CQecTgT7; JB0kBx`~!$h%q#!^ diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index aa9ff56..0d1fc13 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -1,5 +1,8 @@ from functools import lru_cache + from ..core.config import settings +from ..services.grounded_ai_service import GroundedAIService +from ..services.llm_service import LLMService from ..services.rag_service import RagService from ..services.recipe_service import RecipeService @@ -12,5 +15,20 @@ def get_rag_service() -> RagService: ) +@lru_cache +def get_llm_service() -> LLMService: + return LLMService( + api_key=settings.OPENAI_API_KEY, + model=settings.OPENAI_MODEL, + temperature=settings.OPENAI_TEMPERATURE, + max_tokens=settings.OPENAI_MAX_TOKENS, + timeout_seconds=settings.OPENAI_TIMEOUT_SEC, + ) + + +def get_grounded_ai_service() -> GroundedAIService: + return GroundedAIService(rag=get_rag_service(), llm=get_llm_service()) + + def get_recipe_service() -> RecipeService: - return RecipeService(rag=get_rag_service()) \ No newline at end of file + return RecipeService(rag=get_rag_service()) diff --git a/backend/app/api/v1/__pycache__/router.cpython-313.pyc b/backend/app/api/v1/__pycache__/router.cpython-313.pyc deleted file mode 100644 index 0ab89044da8f116a58f11c85fb7df580f098eb09..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 514 zcmey&%ge<81iisaGZh&b7#@Q-Fu)3Be69jArZNOG1T%Uwcrg|+Dlh~y1v7gyc`+BU zK-sK7Hk$%N45Jc*DO6h#dklM!Ae4hd#jpgk=rabhS~3CEV^_fz%m!1D&aTPv5~N#` z@fN3JfM-yCX-R6)OQ2gpB5Xxq?k%>A)Wn>U44?>0Nn(2OE%u_+^HK`(GxJJ{^&onRK;FE?3*p2=%@ahZf|_)T56XZk;Q|^7WAkO^ zCFhi;q(b>C89oEeWVq$-Y!wq)oLW>Ilbl$TlNjTYTAW>yUl8M$Us{w|oZ;w+#7as` z&IVc3$8Nt8= zZ}yG@1Qwu@#pjmXsFXV#?26$xsBc|1%IX+zNEIiU}=FEh>&l zPAtkvjB!aV&MwI>i1EuWEy^s;aP&lCB_$?jr{<-^Bo-6^ab`@JVGNL4ke``XQXB(u zj$T3KEj9?V2;>MhATH(w5)BLwctj?oT;P_zz#?750Tl7mWWB|nmI!oLL1q!i)wkG+ zz<#^MoSs?&)(v(iSaA_2NCAgUZhlH>PO4oIACL_bkWZs`WL h4{Quf9PLt#QXQU`8O1&_F*3@0W@ca#2eZJY0RX8uW@Z2Y diff --git a/backend/app/api/v1/endpoints/__pycache__/recipes.cpython-313.pyc b/backend/app/api/v1/endpoints/__pycache__/recipes.cpython-313.pyc deleted file mode 100644 index 6cbb193186e635b8fca546adb80eeb376edbf3cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 960 zcmZ8fO=}ZD7@nEk>~7NTR@-8s(HcrBM5G_kQY%&{t!NVnk|lU>AgsyQrcJZ!%%)IJ z@#Mj){RJYlC;x(;yl(NJEJ_tmy`{t-;5*q(tPZ>HJnzT6-(Ay8AvkaCclDbDp2+#C{56+!y69WU}6{C z)1uqrb+Ns+c>z6js=F{1>UbXfb-KSmf%V*a(-$=E* z8uJ8~I~`8M^ANL9kF*<1Wm- z)nNDdRbb^b#bY2H1!H`QZhV*D(kXiQGdcP?-PSr9BF3xq%XDB)1#|1|bbvR`H8eF7 j%q)MtcKbY$zzbi^Oh;?V GroundedRecommendationResponse: + try: + return service.grounded_recommendations(payload) + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc diff --git a/backend/app/api/v1/endpoints/health.py b/backend/app/api/v1/endpoints/health.py index 924a534..33f0cf5 100644 --- a/backend/app/api/v1/endpoints/health.py +++ b/backend/app/api/v1/endpoints/health.py @@ -1,8 +1,54 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends, Response, status +import os + +from ...deps import get_llm_service, get_rag_service +from ....core.config import settings +from ....services.llm_service import LLMService +from ....services.rag_service import RagService + router = APIRouter() +def _eval_assets() -> dict: + return { + "retrieval_eval": os.path.exists("eval/datasets/retrieval_eval.json"), + "grounded_generation_eval": os.path.exists("eval/datasets/grounded_generation_eval.json"), + } + + @router.get("/health") def health(): - return {"status": "ok"} \ No newline at end of file + return {"status": "ok", "app": settings.APP_NAME, "version": settings.APP_VERSION} + + +@router.get("/ready") +def ready( + response: Response, + rag: RagService = Depends(get_rag_service), + llm: LLMService = Depends(get_llm_service), +): + rag_status = rag.status() + llm_configured = llm.is_configured() + eval_assets = _eval_assets() + + ready_flag = rag_status["available"] and not rag_status["artifact_mismatch"] + if not ready_flag: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + + return { + "ready": ready_flag, + "rag": rag_status, + "openai_configured": llm_configured, + "openai_model": llm.model, + "eval_assets": eval_assets, + } + + +@router.get("/status") +def status_endpoint( + response: Response, + rag: RagService = Depends(get_rag_service), + llm: LLMService = Depends(get_llm_service), +): + return ready(response=response, rag=rag, llm=llm) diff --git a/backend/app/api/v1/endpoints/meal_plans.py b/backend/app/api/v1/endpoints/meal_plans.py new file mode 100644 index 0000000..5bf1056 --- /dev/null +++ b/backend/app/api/v1/endpoints/meal_plans.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter, Depends, HTTPException + +from ...deps import get_grounded_ai_service +from ....schemas.ai import MealPlanRequest, MealPlanResponse +from ....services.grounded_ai_service import GroundedAIService + +router = APIRouter() + + +@router.post("/meal-plans/generate", response_model=MealPlanResponse) +def generate_meal_plan( + payload: MealPlanRequest, + service: GroundedAIService = Depends(get_grounded_ai_service), +) -> MealPlanResponse: + try: + return service.generate_meal_plan(payload) + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc diff --git a/backend/app/api/v1/endpoints/nutrition.py b/backend/app/api/v1/endpoints/nutrition.py new file mode 100644 index 0000000..d3bf0a0 --- /dev/null +++ b/backend/app/api/v1/endpoints/nutrition.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter, Depends, HTTPException + +from ...deps import get_grounded_ai_service +from ....schemas.ai import NutritionQARequest, NutritionQAResponse +from ....services.grounded_ai_service import GroundedAIService + +router = APIRouter() + + +@router.post("/nutrition/qa", response_model=NutritionQAResponse) +def nutrition_qa( + payload: NutritionQARequest, + service: GroundedAIService = Depends(get_grounded_ai_service), +) -> NutritionQAResponse: + try: + return service.answer_nutrition_question(payload) + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc diff --git a/backend/app/api/v1/endpoints/recipes.py b/backend/app/api/v1/endpoints/recipes.py index 1413662..0903a0a 100644 --- a/backend/app/api/v1/endpoints/recipes.py +++ b/backend/app/api/v1/endpoints/recipes.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, Depends + from ...deps import get_recipe_service from ....services.recipe_service import RecipeService from ....schemas.recipe import RecipeSearchRequest, RecipeSearchResponse @@ -10,6 +11,13 @@ def search_recipes( payload: RecipeSearchRequest, service: RecipeService = Depends(get_recipe_service), -): - results = service.search_recipes(query=payload.query, top_k=payload.top_k) - return RecipeSearchResponse(results=results) \ No newline at end of file +) -> RecipeSearchResponse: + results, diagnostics = service.search_recipes( + query=payload.query, + top_k=payload.top_k, + filters=payload.filters, + rerank=payload.rerank, + include_diagnostics=payload.include_diagnostics, + candidate_k=payload.candidate_k, + ) + return RecipeSearchResponse(results=results, diagnostics=diagnostics) diff --git a/backend/app/api/v1/endpoints/recommend.py b/backend/app/api/v1/endpoints/recommend.py new file mode 100644 index 0000000..e934fba --- /dev/null +++ b/backend/app/api/v1/endpoints/recommend.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter, Depends + +from ...deps import get_recipe_service +from ....schemas.recipe import RecommendRequest, RecipeSearchResponse +from ....services.recipe_service import RecipeService + +router = APIRouter() + + +@router.post("/recipes/recommend", response_model=RecipeSearchResponse) +def recommend_recipes( + payload: RecommendRequest, + service: RecipeService = Depends(get_recipe_service), +) -> RecipeSearchResponse: + results, diagnostics = service.recommend_recipes( + ingredients=payload.ingredients, + dietary_notes=payload.dietary_notes, + top_k=payload.top_k, + filters=payload.filters, + rerank=payload.rerank, + include_diagnostics=payload.include_diagnostics, + candidate_k=payload.candidate_k, + ) + return RecipeSearchResponse(results=results, diagnostics=diagnostics) diff --git a/backend/app/api/v1/endpoints/retrieval.py b/backend/app/api/v1/endpoints/retrieval.py new file mode 100644 index 0000000..4487d2f --- /dev/null +++ b/backend/app/api/v1/endpoints/retrieval.py @@ -0,0 +1,15 @@ +from fastapi import APIRouter, Depends + +from ...deps import get_retrieval_service +from ....schemas.retrieval import RetrievalRequest, RetrievalResponse +from ....services.retrieval_service import RetrievalService + +router = APIRouter() + + +@router.post("/retrieval/search", response_model=RetrievalResponse) +def retrieval_search( + payload: RetrievalRequest, + service: RetrievalService = Depends(get_retrieval_service), +): + return service.search(payload) diff --git a/backend/app/api/v1/router.py b/backend/app/api/v1/router.py index c7a97bc..5f8c7c5 100644 --- a/backend/app/api/v1/router.py +++ b/backend/app/api/v1/router.py @@ -1,7 +1,16 @@ from fastapi import APIRouter + +from .endpoints.grounded_recommend import router as grounded_recommend_router from .endpoints.health import router as health_router +from .endpoints.meal_plans import router as meal_plans_router +from .endpoints.nutrition import router as nutrition_router from .endpoints.recipes import router as recipes_router +from .endpoints.recommend import router as recommend_router api_router = APIRouter() api_router.include_router(health_router, tags=["health"]) -api_router.include_router(recipes_router, tags=["recipes"]) \ No newline at end of file +api_router.include_router(recipes_router, tags=["recipes"]) +api_router.include_router(recommend_router, tags=["recipes"]) +api_router.include_router(grounded_recommend_router, tags=["recipes"]) +api_router.include_router(meal_plans_router, tags=["meal-plans"]) +api_router.include_router(nutrition_router, tags=["nutrition"]) diff --git a/backend/app/core/__pycache__/config.cpython-313.pyc b/backend/app/core/__pycache__/config.cpython-313.pyc deleted file mode 100644 index 5c3ac13fc6974e290d6438ba6753ca436f9defa2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1275 zcmYjROKTfP9G{g|58IL@+fu5S7*w0INTsrKa9f8GP^6W!wjv2lt(taWy4oG7i&wkq z%*r^zQ1w(pFS!JI=pmOv3;h877&gAd1Emc;8gr%T!Llq$JS$Lf|X!^=r5RI#va8WVqin^p7& zXI}sf5D5YjfD#Gb2b#=l2*4necLF2?LnN$4v?vUJ$ZfSbu8Y7Zoa4HAUp_yRFTmK2 zL>7E$d?sCl3Ew*A%acM5mq^^_mVIu8Ec#q(=97kZe4m6bf6uqdkYx96Zk0u+O}-Ib z4CMlLj#=31IxfL{B2mtwj%8yO!Y3Zlhw+*_AQt_$sAh`|l|_q<2C5aGDlBN?<6)A+ zDfRR&AK+=9xF!z~F>yy9*dALjF*S(Q_bk_8L8aDWvRbRua}rtPso7k$zKg0#OQ|v` zKfG6Rdp-SQiu*dzJ#1!OV)3Yy@7X+ymFi81>P>Z5t-;}P&o!}4H#@pTX~A+#e8Q5I zq6#5WYh~pSHHu$687@t%2sW%fF7%IVmTW2|wV^=N*suCV?EME@?vdL(>@~kC?|QWI z8QwO^C$Opv_rB~5Dm#un+$t067-u`KZDNu&T*uQbhxePkl|K7X_I8@xa#L+7s8X+% zl_pZlzk^zCiA51|^d3ft%_GE%8`y%5A@toqwZZEhAfB7w zOA-^JAoU32*1F@ko-RgAxwSY`k`%BkASECz;EsR{2bM;NADCwuNcTKqwFe#+o_D$K zhJS!5#zXCZVnQ`TC$_GYF+K9!zBaoEx2U#m9AU@Q^nPD6_zmJ9N-XrBkqD3FgQ$sq z&*4XKwQ~Qr<3A5CRw|?FKdFt;lQZK|-ndGnem*$gdVG;6jY?-LmvZSUz54Q#i*$aZ zp3zG=|2nhw(m4P0;3Cr+RiAHM%B{Cyu=e0wPCe)TTd(eIPRPycC|FxR&p&-t_~tq& zt+gdz@gI;vZHcT;E2`xpY^ndu)EzMfI(h#@{4mDzj!u>1fr?gGbY>?;H4e7nD?T5e tFo3!5c~B9AeQ>+;SW(i#{YQ%X;T0I diff --git a/backend/app/core/__pycache__/lifespan.cpython-313.pyc b/backend/app/core/__pycache__/lifespan.cpython-313.pyc deleted file mode 100644 index 024cdd6eefa09bf1488684e6e6b8c3f20d7381cd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1298 zcmah}&2Q936o2;EyJH77sptl}QOGo?(5=cAp+=OHqHg*@$yQs4l~XK5#yfW07~2|q zf#g!!OW{N>NV!)%^v_66JtiepBts++?IE{Ri>gZPfj8ORkfKVw!_52s_&tw1hYy>8 z^w-=^flUCON}*3E?@=ej>poauDGd7e;bEp@6 z;2H<3Frjo6qG+pD(bnEqEn0!EwTYgepuSQycBIs-D)|v7U5Z>cjxv5LyY57e&jqTB zpF3$*zjAp;W<&Wk^zZ_|E%)}5u^)FkP|0K++N+iGCH0_1?XHv?FnEw z-Q2#t-wA^`bm`KJjChGGr9TUUzA2aj!c2e8T5HJFC2e9<>E%*@*ISO6D@icC-aNT z<1_V3tY>72;9=~vc#EZN*X3#IwZm{}`Ly)u*_h`9A#c6Qw zdn;WYu@>6dUQ4a`X7t~_s#FxPeqNBD3KzK+L|&XzCkbjTo}@x`!^Rh@;y3@EqW-Iqo8l zS~h;8wi^p>hk?h_#EELjQcnAVIEuODQ%%3YV+U@-?^=0NJNig7zi)ioSRZ*T3 zRR2oZrnM)f(FdimJLEAPxM$uqe>l5x^^Ung#~#try@9&}>#u*XNzXr{pqu;gr+?FG ze?K$eP`LvNWwCb&p8aP3Hgu7iyppagkMMiySp*)oB48Y zJ)$I)a{Qm6gC~e#JaT&BdfaM<{GvFHqI|sR1w61rMR^FNZFpxJM*e`4kBy-%qw=d! iS^4mRaq4zqw*YE+R~=N|TPbz`uMR=QpHd-4F~M_%Hzx%E diff --git a/backend/app/core/__pycache__/logging.cpython-313.pyc b/backend/app/core/__pycache__/logging.cpython-313.pyc deleted file mode 100644 index b78e9f3d3dbdaa3b0a8659c93854d006a2b71b25..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 827 zcmYjO&ubG=5PqBeku)*UMoU$M6rr^1p;9!2N>C*BmklXov6n2A*UjrTcC)+8dn?wH ze}uhx^-vHz3xfZGZ4Qywc=BRzLh$Z;Nw(X8o%!b5dEX2(!+buAgjKCiotlEsZ<)lF zntZ=3;D2dA@q=`@C+r4b$pjP zl8!AcfGWP>AK{KsE9Ox%ZhOffbRV_!I87>BM$9!cF-l@Hm*5zcCm_*iUW@-Fu`K5j zW+_+d3S;i77gODc^JXzdNi53+2zw}2wIY_}B2#U=Tbf_Q)M1@2G3ng=qDS5k&&QMO z5_Acr*cBzF%JoqVH}(VQV#YHE*l&9TsF~$yFK}J4Tb|p)wBytQf4}4Mv`ZLf3_ugz z`BZ&-eYd{4vs&kR6ov%2w%J(Q=6Q<&!QG9q8dvENef|SN`>_oG diff --git a/backend/app/core/config.py b/backend/app/core/config.py index e700c0d..741d6a4 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -6,19 +6,53 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", extra="ignore") APP_NAME: str = "NourishAI API" + APP_VERSION: str = "0.2.0" ENV: str = Field(default="dev", description="dev|staging|prod") LOG_LEVEL: str = Field(default="INFO") + STRICT_STARTUP: bool = Field(default=False, description="Fail startup if required artifacts are missing") CORS_ORIGINS: str = Field(default="*", description="Comma-separated origins or *") - # local file paths for retrieval artifacts + DATA_DIR: str = "data" + ARTIFACTS_DIR: str = "artifacts" + + EMBEDDING_MODEL_NAME: str = "sentence-transformers/all-MiniLM-L6-v2" + EMBEDDING_DIM: int = 384 + RETRIEVAL_CANDIDATES: int = 50 + HYBRID_ALPHA: float = 0.65 + + QDRANT_URL: str = "http://localhost:6333" + QDRANT_COLLECTION: str = "nourishai_chunks" + + LEXICAL_INDEX_PATH: str = "artifacts/lexical_index.pkl" + CHUNK_STORE_PATH: str = "artifacts/chunks.jsonl" + INGESTION_MANIFEST_PATH: str = "artifacts/ingestion_manifest.json" + + MODELS_DIR: str = "models" + CATEGORY_DICT_PATH: str = "models/category_dict.npy" + RAW_RECIPES_PATH: str = "models/raw_recipes.npy" + IMAGE_MODEL_PATH: str = "models/490Image.pkl" + IMAGE_MODEL_URL: str = "https://github.com/CarlAmine/EECE-490/releases/download/v1.0/490Image.pkl" + IMAGE_MODEL_EXPECTED_SIZE_MB: int = 687 + RECIPES_DATA_PATH: str = "models/recipes_data.pkl" + FAISS_INDEX_PATH: str = "models/faiss.index" RECIPES_PKL_PATH: str = "models/recipe.pkl" - # when artifacts are missing the service will try to download everything - # from this Google Drive folder ID via gdown - GDRIVE_FOLDER_ID: str = Field(default="10okoRXmRZDGtsF8e5cDxSLEzJWfuF9nl", - description="Drive folder containing faiss.index + recipe.pkl") + OPENAI_API_KEY: str | None = None + OPENAI_MODEL: str = "gpt-4o-mini" + OPENAI_TEMPERATURE: float = 0.2 + OPENAI_MAX_TOKENS: int = 800 + OPENAI_TIMEOUT_SEC: float = 20.0 + + GDRIVE_RAW_RECIPES_ID: str = Field( + default="1lE6zl-9dJKUnNu6CDuBvHPpGodKrGnoX", + description="Drive file containing raw_recipes.npy", + ) + GDRIVE_FOLDER_ID: str = Field( + default="10okoRXmRZDGtsF8e5cDxSLEzJWfuF9nl", + description="Drive folder containing faiss.index + recipe.pkl", + ) -settings = Settings() \ No newline at end of file +settings = Settings() diff --git a/backend/app/core/lifespan.py b/backend/app/core/lifespan.py index 45a2839..c32e553 100644 --- a/backend/app/core/lifespan.py +++ b/backend/app/core/lifespan.py @@ -3,6 +3,9 @@ from fastapi import FastAPI from .logging import configure_logging +from .observability import log_event +from .validation import validate_startup +from .config import settings logger = logging.getLogger("nourishai") @@ -10,16 +13,25 @@ @asynccontextmanager async def lifespan(app: FastAPI): configure_logging() - logger.info("Starting NourishAI API") - # initialize shared resources; failure to load RAG should not prevent boot + log_event(logger, "startup_begin") try: - from ..api.deps import get_rag_service + from ..api.deps import get_llm_service, get_rag_service - # instantiate service to trigger artifact loading only; do not call `search` - _ = get_rag_service() - logger.info("RAG service initialized (artifacts may or may not be present)") + rag = get_rag_service() + llm = get_llm_service() + log_event(logger, "startup_services_initialized") + + report = validate_startup(rag, llm) + for warning in report["warnings"]: + log_event(logger, "startup_warning", detail=warning) + for error in report["errors"]: + log_event(logger, "startup_error", detail=error) + if settings.STRICT_STARTUP and report["errors"]: + raise RuntimeError("Startup validation failed: " + "; ".join(report["errors"])) except Exception as exc: # pragma: no cover - artifacts may be absent - logger.exception("RAG init failed, continuing without it: %s", exc) + log_event(logger, "startup_exception", error=str(exc)) + if settings.STRICT_STARTUP: + raise yield - logger.info("Shutting down NourishAI API") \ No newline at end of file + log_event(logger, "shutdown") diff --git a/backend/app/core/logging.py b/backend/app/core/logging.py index 24b3341..2e9fc07 100644 --- a/backend/app/core/logging.py +++ b/backend/app/core/logging.py @@ -1,11 +1,79 @@ +from __future__ import annotations + +import json import logging import sys +from datetime import datetime, timezone +from typing import Any, Dict + from .config import settings +from .observability import get_request_id + + +_STANDARD_ATTRS = { + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "message", +} + + +class RequestIdFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + record.request_id = get_request_id() or "-" + return True + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: Dict[str, Any] = { + "timestamp": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "request_id": getattr(record, "request_id", "-"), + } + + event = getattr(record, "event", None) + if event: + payload["event"] = event + + extras = { + key: value + for key, value in record.__dict__.items() + if key not in _STANDARD_ATTRS and key not in {"event", "request_id"} + } + if extras: + payload["fields"] = extras + + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + + return json.dumps(payload, default=str) def configure_logging() -> None: - logging.basicConfig( - level=getattr(logging, settings.LOG_LEVEL.upper(), logging.INFO), - format="%(asctime)s %(levelname)s %(name)s %(message)s", - handlers=[logging.StreamHandler(sys.stdout)], - ) \ No newline at end of file + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + handler.addFilter(RequestIdFilter()) + + root = logging.getLogger() + root.setLevel(getattr(logging, settings.LOG_LEVEL.upper(), logging.INFO)) + root.handlers = [handler] diff --git a/backend/app/core/observability.py b/backend/app/core/observability.py new file mode 100644 index 0000000..ea9467e --- /dev/null +++ b/backend/app/core/observability.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import contextvars +import logging +import time +import uuid +from typing import Any, Optional + +from starlette.requests import Request +from starlette.responses import Response + + +REQUEST_ID_HEADER = "X-Request-ID" + +_request_id_ctx: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("request_id", default=None) + + +def set_request_id(request_id: str) -> None: + _request_id_ctx.set(request_id) + + +def get_request_id() -> Optional[str]: + return _request_id_ctx.get() + + +def clear_request_id() -> None: + _request_id_ctx.set(None) + + +def log_event(logger: logging.Logger, event: str, **fields: Any) -> None: + logger.info(event, extra={"event": event, **fields}) + + +async def request_context_middleware(request: Request, call_next) -> Response: + logger = logging.getLogger("nourishai.request") + request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) + set_request_id(request_id) + start = time.perf_counter() + + log_event( + logger, + "request_received", + method=request.method, + path=request.url.path, + query=request.url.query, + client=request.client.host if request.client else None, + ) + + try: + response = await call_next(request) + except Exception as exc: + latency_ms = (time.perf_counter() - start) * 1000.0 + logger.exception( + "request_failed", + extra={ + "event": "request_failed", + "method": request.method, + "path": request.url.path, + "latency_ms": round(latency_ms, 2), + "error": str(exc), + }, + ) + clear_request_id() + raise + + latency_ms = (time.perf_counter() - start) * 1000.0 + log_event( + logger, + "request_completed", + method=request.method, + path=request.url.path, + status_code=response.status_code, + latency_ms=round(latency_ms, 2), + ) + + response.headers[REQUEST_ID_HEADER] = request_id + clear_request_id() + return response diff --git a/backend/app/core/validation.py b/backend/app/core/validation.py new file mode 100644 index 0000000..c22e08d --- /dev/null +++ b/backend/app/core/validation.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from ..services.llm_service import LLMService +from ..services.rag_service import RagService +from .config import settings + + +def validate_startup(rag: Optional[RagService], llm: Optional[LLMService]) -> Dict[str, Any]: + errors: List[str] = [] + warnings: List[str] = [] + + rag_status = rag.status() if rag else { + "index_loaded": False, + "recipes_loaded": False, + "index_count": 0, + "recipe_count": 0, + "artifact_mismatch": False, + "available": False, + } + + if not rag_status["index_loaded"]: + errors.append(f"FAISS index is not loaded at {settings.FAISS_INDEX_PATH}.") + if not rag_status["recipes_loaded"]: + errors.append(f"Recipe metadata is not loaded at {settings.RECIPES_PKL_PATH}.") + if rag_status["artifact_mismatch"]: + warnings.append("FAISS index count does not match recipe count; rebuild artifacts.") + + llm_configured = llm.is_configured() if llm else False + if not llm_configured: + warnings.append("OpenAI API key is not configured; grounded generation endpoints will be limited.") + + return { + "errors": errors, + "warnings": warnings, + "rag_status": rag_status, + "llm_configured": llm_configured, + } diff --git a/backend/app/ingestion/__init__.py b/backend/app/ingestion/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/app/ingestion/__init__.py @@ -0,0 +1 @@ + diff --git a/backend/app/ingestion/chunking.py b/backend/app/ingestion/chunking.py new file mode 100644 index 0000000..2198bc0 --- /dev/null +++ b/backend/app/ingestion/chunking.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import re +from typing import Dict, List + + +def _as_list(value) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v).strip() for v in value if str(v).strip()] + if isinstance(value, str): + if "," in value: + return [v.strip() for v in value.split(",") if v.strip()] + return [value.strip()] + return [str(value).strip()] + + +def chunk_recipe_texts(recipe: Dict[str, object]) -> List[Dict[str, str]]: + title = str(recipe.get("title") or recipe.get("name") or "").strip() + ingredients = _as_list(recipe.get("ingredients") or recipe.get("ingredient_lines")) + steps = _as_list(recipe.get("instructions") or recipe.get("steps") or recipe.get("directions")) + + chunks: List[Dict[str, str]] = [] + if ingredients: + ing_text = "Ingredients:\n" + "\n".join(f"- {item}" for item in ingredients) + chunks.append({"section": "ingredients", "text": _prefix_title(title, ing_text)}) + + if steps: + step_groups = _group_steps(steps) + for idx, group in enumerate(step_groups, start=1): + step_text = "Instructions (part {idx}):\n".format(idx=idx) + "\n".join( + f"{i+1}. {step}" for i, step in enumerate(group) + ) + chunks.append({"section": f"instructions_{idx}", "text": _prefix_title(title, step_text)}) + + if not chunks and title: + chunks.append({"section": "summary", "text": f"Recipe: {title}."}) + + return chunks + + +def chunk_text_by_paragraphs(text: str, max_words: int = 220, min_words: int = 80) -> List[str]: + cleaned = (text or "").strip() + if not cleaned: + return [] + + paragraphs = [p.strip() for p in re.split(r"\n\s*\n", cleaned) if p.strip()] + if not paragraphs: + return [] + + chunks: List[str] = [] + current: List[str] = [] + current_words = 0 + + for paragraph in paragraphs: + word_count = _word_count(paragraph) + if current_words + word_count > max_words and current_words >= min_words: + chunks.append("\n\n".join(current)) + current = [paragraph] + current_words = word_count + else: + current.append(paragraph) + current_words += word_count + + if current: + chunks.append("\n\n".join(current)) + + return chunks + + +def _group_steps(steps: List[str], max_steps: int = 4, max_words: int = 180) -> List[List[str]]: + groups: List[List[str]] = [] + current: List[str] = [] + current_words = 0 + + for step in steps: + step_words = _word_count(step) + if current and (len(current) >= max_steps or current_words + step_words > max_words): + groups.append(current) + current = [step] + current_words = step_words + else: + current.append(step) + current_words += step_words + + if current: + groups.append(current) + + return groups + + +def _word_count(text: str) -> int: + return len(re.findall(r"\b\w+\b", text)) + + +def _prefix_title(title: str, text: str) -> str: + if not title: + return text + return f"Recipe: {title}\n{text}" diff --git a/backend/app/ingestion/cli.py b/backend/app/ingestion/cli.py new file mode 100644 index 0000000..454074b --- /dev/null +++ b/backend/app/ingestion/cli.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import argparse +import json + +from .pipeline import run_ingestion +from ..core.config import settings + + +def main() -> None: + parser = argparse.ArgumentParser(description="NourishAI ingestion pipeline") + parser.add_argument("--data-dir", default=settings.DATA_DIR, help="Data directory containing recipes/nutrition/ingredients") + parser.add_argument("--recreate", action="store_true", help="Drop and recreate the vector collection") + parser.add_argument("--batch-size", type=int, default=64, help="Embedding batch size") + args = parser.parse_args() + + report = run_ingestion(data_dir=args.data_dir, recreate=args.recreate, batch_size=args.batch_size) + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/backend/app/ingestion/loaders.py b/backend/app/ingestion/loaders.py new file mode 100644 index 0000000..6f4e109 --- /dev/null +++ b/backend/app/ingestion/loaders.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any, Dict, List + +logger = logging.getLogger("nourishai.ingestion") + + +def load_recipe_records(data_dir: str) -> List[Dict[str, Any]]: + return _load_records(Path(data_dir) / "recipes", "recipes") + + +def load_nutrition_records(data_dir: str) -> List[Dict[str, Any]]: + return _load_records(Path(data_dir) / "nutrition", "nutrition") + + +def load_ingredient_records(data_dir: str) -> List[Dict[str, Any]]: + return _load_records(Path(data_dir) / "ingredients", "ingredients") + + +def _load_records(folder: Path, label: str) -> List[Dict[str, Any]]: + if not folder.exists(): + logger.warning("%s data folder not found: %s", label, folder) + return [] + + records: List[Dict[str, Any]] = [] + for path in sorted(folder.glob("*.json")): + try: + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + except Exception as exc: + logger.warning("Failed to load %s: %s", path, exc) + continue + + if isinstance(payload, list): + records.extend(payload) + elif isinstance(payload, dict): + if isinstance(payload.get("items"), list): + records.extend(payload["items"]) + else: + records.append(payload) + else: + logger.warning("Unsupported payload format in %s", path) + + logger.info("Loaded %d %s records", len(records), label) + return records diff --git a/backend/app/ingestion/models.py b/backend/app/ingestion/models.py new file mode 100644 index 0000000..ed0b0b7 --- /dev/null +++ b/backend/app/ingestion/models.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict + + +@dataclass(frozen=True) +class DocumentChunk: + chunk_id: str + doc_id: str + doc_type: str + text: str + payload: Dict[str, Any] diff --git a/backend/app/ingestion/pipeline.py b/backend/app/ingestion/pipeline.py new file mode 100644 index 0000000..5716be8 --- /dev/null +++ b/backend/app/ingestion/pipeline.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import json +import logging +import os +import uuid +from datetime import datetime +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from .chunking import chunk_recipe_texts, chunk_text_by_paragraphs +from .loaders import load_ingredient_records, load_nutrition_records, load_recipe_records +from .models import DocumentChunk +from ..core.config import settings +from ..repositories.lexical_index import LexicalIndex +from ..repositories.vector_store import QdrantVectorStore +from ..services.embedding_service import EmbeddingService + +logger = logging.getLogger("nourishai.ingestion") + + +def run_ingestion( + data_dir: Optional[str] = None, + recreate: bool = False, + batch_size: int = 64, +) -> Dict[str, Any]: + data_root = data_dir or settings.DATA_DIR + vector_store = QdrantVectorStore( + url=settings.QDRANT_URL, + collection_name=settings.QDRANT_COLLECTION, + vector_size=settings.EMBEDDING_DIM, + ) + + if recreate: + vector_store.recreate_collection() + else: + vector_store.ensure_collection() + + recipes = load_recipe_records(data_root) + nutrition_refs = load_nutrition_records(data_root) + ingredients = load_ingredient_records(data_root) + + chunks: List[DocumentChunk] = [] + for record in recipes: + chunks.extend(_chunks_from_recipe(record)) + for record in nutrition_refs: + chunks.extend(_chunks_from_nutrition(record)) + for record in ingredients: + chunks.extend(_chunks_from_ingredient(record)) + + logger.info("Prepared %d chunks for ingestion", len(chunks)) + + if not chunks: + return { + "status": "no_data", + "chunks": 0, + "recipes": len(recipes), + "nutrition": len(nutrition_refs), + "ingredients": len(ingredients), + } + + embedder = EmbeddingService(settings.EMBEDDING_MODEL_NAME) + + for batch in _batch_iter(chunks, batch_size): + texts = [chunk.text for chunk in batch] + vectors = embedder.embed(texts) + vector_store.upsert(batch, vectors) + + _write_chunks_store(chunks, settings.CHUNK_STORE_PATH) + + lexical_index = LexicalIndex(settings.LEXICAL_INDEX_PATH) + lexical_index.build(chunks) + + manifest = { + "timestamp": datetime.utcnow().isoformat() + "Z", + "collection": settings.QDRANT_COLLECTION, + "embedding_model": settings.EMBEDDING_MODEL_NAME, + "counts": { + "recipes": len(recipes), + "nutrition": len(nutrition_refs), + "ingredients": len(ingredients), + "chunks": len(chunks), + }, + } + _write_manifest(manifest, settings.INGESTION_MANIFEST_PATH) + return {"status": "ok", **manifest} + + +def _chunks_from_recipe(record: Dict[str, Any]) -> List[DocumentChunk]: + title = str(record.get("title") or record.get("name") or "").strip() + doc_id = str(record.get("id") or record.get("recipe_id") or _slugify(title) or uuid.uuid4().hex) + meal_type = _normalize_tags(record.get("meal_type") or record.get("course")) + dietary_tags = _normalize_tags(record.get("dietary_tags") or record.get("diet") or record.get("tags")) + cuisine = _normalize_tags(record.get("cuisine") or record.get("cuisine_type")) + ingredients = _as_list(record.get("ingredients") or record.get("ingredient_lines")) + instructions = _as_list(record.get("instructions") or record.get("steps") or record.get("directions")) + + nutrition = record.get("nutrition") or {} + calories = _coerce_float(record.get("calories") or nutrition.get("calories")) + protein_g = _coerce_float(nutrition.get("protein_g") or nutrition.get("protein")) + carbs_g = _coerce_float(nutrition.get("carbs_g") or nutrition.get("carbohydrates")) + fat_g = _coerce_float(nutrition.get("fat_g") or nutrition.get("fat")) + + base_payload = { + "doc_id": doc_id, + "doc_type": "recipe", + "title": title, + "meal_type": meal_type, + "dietary_tags": dietary_tags, + "cuisine": cuisine, + "ingredients": ingredients, + "instructions": instructions, + "calories": calories, + "protein_g": protein_g, + "carbs_g": carbs_g, + "fat_g": fat_g, + "source": record.get("source"), + } + + chunks: List[DocumentChunk] = [] + for chunk in chunk_recipe_texts(record): + chunk_id = uuid.uuid4().hex + payload = dict(base_payload) + payload["chunk_section"] = chunk["section"] + chunks.append( + DocumentChunk( + chunk_id=chunk_id, + doc_id=doc_id, + doc_type="recipe", + text=chunk["text"], + payload=payload, + ) + ) + + return chunks + + +def _chunks_from_nutrition(record: Dict[str, Any]) -> List[DocumentChunk]: + title = str(record.get("title") or record.get("name") or "").strip() + doc_id = str(record.get("id") or record.get("ref_id") or _slugify(title) or uuid.uuid4().hex) + text = str(record.get("text") or record.get("description") or "").strip() + nutrients = _normalize_tags(record.get("nutrients") or record.get("nutrient")) + source = record.get("source") + + chunks: List[DocumentChunk] = [] + for idx, chunk_text in enumerate(chunk_text_by_paragraphs(text)): + chunk_id = uuid.uuid4().hex + payload = { + "doc_id": doc_id, + "doc_type": "nutrition", + "title": title, + "nutrients": nutrients, + "source": source, + "chunk_section": f"paragraph_{idx+1}", + } + chunks.append( + DocumentChunk( + chunk_id=chunk_id, + doc_id=doc_id, + doc_type="nutrition", + text=_prefix_title(title, chunk_text), + payload=payload, + ) + ) + + return chunks + + +def _chunks_from_ingredient(record: Dict[str, Any]) -> List[DocumentChunk]: + name = str(record.get("name") or record.get("ingredient") or "").strip() + doc_id = str(record.get("id") or record.get("ingredient_id") or _slugify(name) or uuid.uuid4().hex) + aliases = _normalize_tags(record.get("aliases") or record.get("aka")) + notes = str(record.get("notes") or record.get("description") or "").strip() + substitutions = _as_list(record.get("substitutions") or record.get("substitute")) + allergens = _normalize_tags(record.get("allergens")) + nutrition = record.get("nutrition") or {} + + calories = _coerce_float(nutrition.get("calories")) + protein_g = _coerce_float(nutrition.get("protein_g") or nutrition.get("protein")) + carbs_g = _coerce_float(nutrition.get("carbs_g") or nutrition.get("carbohydrates")) + fat_g = _coerce_float(nutrition.get("fat_g") or nutrition.get("fat")) + + text = _build_ingredient_text(name, aliases, notes, substitutions, nutrition) + chunks: List[DocumentChunk] = [] + for idx, chunk_text in enumerate(chunk_text_by_paragraphs(text)): + chunk_id = uuid.uuid4().hex + payload = { + "doc_id": doc_id, + "doc_type": "ingredient", + "title": name, + "aliases": aliases, + "allergens": allergens, + "calories": calories, + "protein_g": protein_g, + "carbs_g": carbs_g, + "fat_g": fat_g, + "source": record.get("source"), + "chunk_section": f"paragraph_{idx+1}", + } + chunks.append( + DocumentChunk( + chunk_id=chunk_id, + doc_id=doc_id, + doc_type="ingredient", + text=chunk_text, + payload=payload, + ) + ) + + return chunks + + +def _write_chunks_store(chunks: Iterable[DocumentChunk], path: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + for chunk in chunks: + handle.write( + json.dumps( + { + "chunk_id": chunk.chunk_id, + "doc_id": chunk.doc_id, + "doc_type": chunk.doc_type, + "text": chunk.text, + "payload": chunk.payload, + } + ) + + "\n" + ) + + +def _write_manifest(manifest: Dict[str, Any], path: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(manifest, handle, indent=2) + + +def _batch_iter(items: List[DocumentChunk], batch_size: int) -> Iterable[List[DocumentChunk]]: + for idx in range(0, len(items), batch_size): + yield items[idx : idx + batch_size] + + +def _normalize_tags(value: Any) -> List[str]: + values = _as_list(value) + return [v.lower() for v in values] + + +def _as_list(value: Any) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v).strip() for v in value if str(v).strip()] + if isinstance(value, str): + if "," in value: + return [v.strip() for v in value.split(",") if v.strip()] + return [value.strip()] + return [str(value).strip()] + + +def _coerce_float(value: Any) -> Optional[float]: + try: + if value is None or value == "": + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _slugify(text: str) -> str: + return "-".join(part for part in (text or "").lower().split() if part) + + +def _prefix_title(title: str, text: str) -> str: + if not title: + return text + return f"{title}\n{text}" + + +def _build_ingredient_text( + name: str, + aliases: List[str], + notes: str, + substitutions: List[str], + nutrition: Dict[str, Any], +) -> str: + lines = [f"Ingredient: {name}"] if name else [] + if aliases: + lines.append("Aliases: " + ", ".join(aliases)) + if notes: + lines.append("Notes: " + notes) + if substitutions: + lines.append("Substitutions: " + ", ".join(substitutions)) + if nutrition: + nutrition_bits = [] + for key in ("calories", "protein_g", "carbs_g", "fat_g"): + if key in nutrition: + nutrition_bits.append(f"{key}={nutrition[key]}") + if nutrition_bits: + lines.append("Nutrition: " + "; ".join(nutrition_bits)) + return "\n".join(lines) diff --git a/backend/app/main.py b/backend/app/main.py index aba2372..194a4b6 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,6 +3,7 @@ from .core.config import settings from .core.lifespan import lifespan +from .core.observability import request_context_middleware from .api.v1.router import api_router @@ -14,7 +15,7 @@ def _parse_origins(origins: str): app = FastAPI( title=settings.APP_NAME, - version="0.1.0", + version=settings.APP_VERSION, lifespan=lifespan, ) @@ -25,5 +26,6 @@ def _parse_origins(origins: str): allow_methods=["*"], allow_headers=["*"], ) +app.middleware("http")(request_context_middleware) -app.include_router(api_router, prefix="/api/v1") \ No newline at end of file +app.include_router(api_router, prefix="/api/v1") diff --git a/backend/app/repositories/lexical_index.py b/backend/app/repositories/lexical_index.py new file mode 100644 index 0000000..3c0a794 --- /dev/null +++ b/backend/app/repositories/lexical_index.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import logging +import os +import pickle +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from rank_bm25 import BM25Okapi + +from ..ingestion.models import DocumentChunk +from ..services.filters import payload_matches + +logger = logging.getLogger("nourishai.lexical") + + +@dataclass(frozen=True) +class LexicalSearchResult: + chunk_id: str + score: float + payload: Dict[str, Any] + text: str + + +class LexicalIndex: + def __init__(self, index_path: str): + self.index_path = index_path + self._bm25: Optional[BM25Okapi] = None + self._chunk_ids: List[str] = [] + self._payloads: List[Dict[str, Any]] = [] + self._texts: List[str] = [] + self._tokenized: List[List[str]] = [] + + @property + def available(self) -> bool: + return self._bm25 is not None + + def build(self, chunks: List[DocumentChunk]) -> None: + self._chunk_ids = [chunk.chunk_id for chunk in chunks] + self._payloads = [chunk.payload for chunk in chunks] + self._texts = [chunk.text for chunk in chunks] + self._tokenized = [self._tokenize(text) for text in self._texts] + self._bm25 = BM25Okapi(self._tokenized) + self._save() + logger.info("Lexical index built with %d chunks", len(self._chunk_ids)) + + def load(self) -> None: + if not os.path.exists(self.index_path): + logger.warning("Lexical index not found: %s", self.index_path) + return + with open(self.index_path, "rb") as handle: + data = pickle.load(handle) + self._chunk_ids = data.get("chunk_ids", []) + self._payloads = data.get("payloads", []) + self._texts = data.get("texts", []) + self._tokenized = data.get("tokenized", []) + if self._tokenized: + self._bm25 = BM25Okapi(self._tokenized) + logger.info("Lexical index loaded (%d chunks)", len(self._chunk_ids)) + + def search( + self, + query: str, + top_k: int, + filters, + doc_types: Optional[List[str]] = None, + ) -> List[LexicalSearchResult]: + if self._bm25 is None: + self.load() + if self._bm25 is None: + return [] + + tokenized_query = self._tokenize(query) + scores = self._bm25.get_scores(tokenized_query) + + candidates: List[LexicalSearchResult] = [] + for idx, score in enumerate(scores): + payload = self._payloads[idx] + if not payload_matches(payload, filters, doc_types): + continue + if score <= 0: + continue + candidates.append( + LexicalSearchResult( + chunk_id=self._chunk_ids[idx], + score=float(score), + payload=payload, + text=self._texts[idx], + ) + ) + + candidates.sort(key=lambda item: item.score, reverse=True) + return candidates[:top_k] + + def _save(self) -> None: + os.makedirs(os.path.dirname(self.index_path), exist_ok=True) + with open(self.index_path, "wb") as handle: + pickle.dump( + { + "chunk_ids": self._chunk_ids, + "payloads": self._payloads, + "texts": self._texts, + "tokenized": self._tokenized, + }, + handle, + ) + + @staticmethod + def _tokenize(text: str) -> List[str]: + return re.findall(r"[a-z0-9]+", text.lower()) diff --git a/backend/app/repositories/vector_store.py b/backend/app/repositories/vector_store.py new file mode 100644 index 0000000..cea70ef --- /dev/null +++ b/backend/app/repositories/vector_store.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import logging +from typing import Iterable, List, Optional + +from qdrant_client import QdrantClient +from qdrant_client.models import ( + Distance, + FieldCondition, + Filter, + MatchAny, + PayloadSchemaType, + PointStruct, + Range, + VectorParams, +) + +from ..ingestion.models import DocumentChunk + +logger = logging.getLogger("nourishai.vector_store") + + +FILTERABLE_FIELDS = { + "doc_type": PayloadSchemaType.KEYWORD, + "meal_type": PayloadSchemaType.KEYWORD, + "dietary_tags": PayloadSchemaType.KEYWORD, + "cuisine": PayloadSchemaType.KEYWORD, + "calories": PayloadSchemaType.FLOAT, + "protein_g": PayloadSchemaType.FLOAT, + "carbs_g": PayloadSchemaType.FLOAT, + "fat_g": PayloadSchemaType.FLOAT, + "allergens": PayloadSchemaType.KEYWORD, + "aliases": PayloadSchemaType.KEYWORD, + "nutrients": PayloadSchemaType.KEYWORD, +} + + +class QdrantVectorStore: + def __init__(self, url: str, collection_name: str, vector_size: int): + self.client = QdrantClient(url=url) + self.collection_name = collection_name + self.vector_size = vector_size + + def ensure_collection(self) -> None: + if not self.client.collection_exists(self.collection_name): + logger.info("Creating Qdrant collection %s", self.collection_name) + self.client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams(size=self.vector_size, distance=Distance.COSINE), + ) + self._ensure_payload_indexes() + + def recreate_collection(self) -> None: + logger.info("Recreating Qdrant collection %s", self.collection_name) + self.client.recreate_collection( + collection_name=self.collection_name, + vectors_config=VectorParams(size=self.vector_size, distance=Distance.COSINE), + ) + self._ensure_payload_indexes() + + def upsert(self, chunks: Iterable[DocumentChunk], vectors: List[List[float]]) -> None: + points: List[PointStruct] = [] + for chunk, vector in zip(chunks, vectors): + payload = dict(chunk.payload) + payload["text"] = chunk.text + payload["chunk_id"] = chunk.chunk_id + points.append(PointStruct(id=chunk.chunk_id, vector=vector, payload=payload)) + + self.client.upsert(collection_name=self.collection_name, points=points) + + def search( + self, + vector: List[float], + limit: int, + qdrant_filter: Optional[Filter], + ): + return self.client.search( + collection_name=self.collection_name, + query_vector=vector, + limit=limit, + query_filter=qdrant_filter, + with_payload=True, + ) + + def _ensure_payload_indexes(self) -> None: + for field_name, schema in FILTERABLE_FIELDS.items(): + try: + self.client.create_payload_index( + collection_name=self.collection_name, + field_name=field_name, + field_schema=schema, + ) + except Exception: + pass + + +def build_range_condition(key: str, min_value: Optional[float], max_value: Optional[float]) -> Optional[FieldCondition]: + if min_value is None and max_value is None: + return None + return FieldCondition(key=key, range=Range(gte=min_value, lte=max_value)) + + +def build_match_any_condition(key: str, values: Optional[List[str]]) -> Optional[FieldCondition]: + if not values: + return None + return FieldCondition(key=key, match=MatchAny(any=values)) diff --git a/backend/app/schemas/__pycache__/recipe.cpython-313.pyc b/backend/app/schemas/__pycache__/recipe.cpython-313.pyc deleted file mode 100644 index 645bf7c92f075c2900e8ff30f492ba02503e78f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1423 zcmZuwOK%)S5bk;I?86D`#8GTs>nIMHLmVjxDMk?}K)~LB5gBnnYH2i??Ol&No>^DV zN;s!TImEtg4v~E15()kd!IHJ~B?nG>L+o;=s%H&0$w>1}^;g|BU0+pA=H_Y)pWlKP z$zKLzf6(IOac0J(i^g*nu)w&_+6EYH6U-K^&HGl{2D|Nm(_)v|6&6@aEU>xpkyV}r zSF;YX?kTpS*$T4WDc0*%{o0rg`lhOW7D@3<-Vv#CzDz{gLC*d;VW?bzQ z=V`7A^`7h3l>0!$NnecVfHISga*Cu#g{mi64@4&sSs_&;$z%b8m@<%?%8R2kheSx_ z$T$b_3-cSw3&Skh5n-rmVYrib1}WK%Fx(wP>CAIJ47&-)B26-p<>0N z_%H;z3!Ecx=~hlR&7%U&lf4jzk~YR+R1}ar927$0iMP-+q=G=Q7?BJCWp|2Q;k|=TkL_hx zJPCKE0VqLwChG{VHvsmg8CZe6W`e_AUI}!OxsALVR01!ku33QPMo&$bwGOpj!=FweNH zzbVn{7z_v~s%#)&x6L(&*ADoxy;x>L?oGd{y#8J%%5VsP($T6ax7h2`$m(BFSC##% z_0zG>3ah+3<&1s;H*hl*R^CMT(=rU>6WoS~HBL M!?nK|e)PTm0nqA9M*si- diff --git a/backend/app/schemas/ai.py b/backend/app/schemas/ai.py new file mode 100644 index 0000000..99bf5ee --- /dev/null +++ b/backend/app/schemas/ai.py @@ -0,0 +1,112 @@ +from pydantic import BaseModel, Field +from typing import List, Optional + +from .recipe import RecipeFilters, RetrievalDiagnostics + + +class MacroSummary(BaseModel): + calories: Optional[float] = None + protein_g: Optional[float] = None + carbs_g: Optional[float] = None + fat_g: Optional[float] = None + + +class SourceRecipeRef(BaseModel): + id: str + title: str + + +class GroundedRecommendationRequest(BaseModel): + ingredients: List[str] = Field(..., description="Ingredients available") + dietary_notes: Optional[str] = Field(default=None) + top_k: int = Field(default=5, ge=1, le=50) + filters: Optional[RecipeFilters] = None + rerank: bool = Field(default=True) + include_diagnostics: bool = Field(default=False) + candidate_k: Optional[int] = Field(default=None, ge=1, le=200) + + +class RecommendedRecipe(BaseModel): + recipe_id: str + title: str + why_selected: str + substitutions: List[str] = Field(default_factory=list) + warnings: List[str] = Field(default_factory=list) + nutrition_summary: Optional[MacroSummary] = None + + +class LLMRecommendationPayload(BaseModel): + recommended_recipes: List[RecommendedRecipe] + warnings: List[str] = Field(default_factory=list) + + +class GroundedRecommendationResponse(BaseModel): + recommended_recipes: List[RecommendedRecipe] + warnings: List[str] = Field(default_factory=list) + source_recipes: List[SourceRecipeRef] = Field(default_factory=list) + retrieval_diagnostics: Optional[RetrievalDiagnostics] = None + + +class MealPlanRequest(BaseModel): + dietary_profile: Optional[str] = None + calorie_target: Optional[int] = Field(default=None, ge=0) + days: int = Field(default=3, ge=1, le=14) + meals_per_day: int = Field(default=3, ge=1, le=5) + meal_preferences: Optional[List[str]] = None + filters: Optional[RecipeFilters] = None + rerank: bool = Field(default=True) + include_diagnostics: bool = Field(default=False) + candidate_k: Optional[int] = Field(default=None, ge=1, le=200) + + +class MealPlanMeal(BaseModel): + recipe_id: str + title: str + meal_type: Optional[str] = None + why_selected: str + estimated_calories: Optional[float] = None + macros: Optional[MacroSummary] = None + warnings: List[str] = Field(default_factory=list) + + +class MealPlanDay(BaseModel): + day: int + meals: List[MealPlanMeal] + estimated_totals: Optional[MacroSummary] = None + notes: Optional[str] = None + + +class LLMMealPlanPayload(BaseModel): + days: List[MealPlanDay] + dietary_notes: Optional[str] = None + warnings: List[str] = Field(default_factory=list) + + +class MealPlanResponse(BaseModel): + days: List[MealPlanDay] + dietary_notes: Optional[str] = None + warnings: List[str] = Field(default_factory=list) + source_recipes: List[SourceRecipeRef] = Field(default_factory=list) + retrieval_diagnostics: Optional[RetrievalDiagnostics] = None + + +class NutritionQARequest(BaseModel): + question: str = Field(..., description="Nutrition question grounded in recipes") + top_k: int = Field(default=5, ge=1, le=50) + filters: Optional[RecipeFilters] = None + rerank: bool = Field(default=True) + include_diagnostics: bool = Field(default=False) + candidate_k: Optional[int] = Field(default=None, ge=1, le=200) + + +class LLMNutritionQAPayload(BaseModel): + answer: str + supporting_recipe_ids: List[str] = Field(default_factory=list) + warnings: List[str] = Field(default_factory=list) + + +class NutritionQAResponse(BaseModel): + answer: str + supporting_sources: List[SourceRecipeRef] = Field(default_factory=list) + warnings: List[str] = Field(default_factory=list) + retrieval_diagnostics: Optional[RetrievalDiagnostics] = None diff --git a/backend/app/schemas/recipe.py b/backend/app/schemas/recipe.py index 3cdf185..a907cb0 100644 --- a/backend/app/schemas/recipe.py +++ b/backend/app/schemas/recipe.py @@ -8,13 +8,67 @@ class Recipe(BaseModel): ingredients: List[str] instructions: List[str] calories: Optional[float] = None + protein_g: Optional[float] = None + carbs_g: Optional[float] = None + fat_g: Optional[float] = None + cuisine: Optional[List[str]] = None + meal_type: Optional[List[str]] = None + dietary_tags: Optional[List[str]] = None + allergens: Optional[List[str]] = None + prep_time_minutes: Optional[int] = None + cook_time_minutes: Optional[int] = None + total_time_minutes: Optional[int] = None score: Optional[float] = None +class RecipeFilters(BaseModel): + cuisine: Optional[List[str]] = None + meal_type: Optional[List[str]] = None + dietary_tags: Optional[List[str]] = None + + min_calories: Optional[float] = None + max_calories: Optional[float] = None + min_protein_g: Optional[float] = None + max_protein_g: Optional[float] = None + min_carbs_g: Optional[float] = None + max_carbs_g: Optional[float] = None + min_fat_g: Optional[float] = None + max_fat_g: Optional[float] = None + + max_prep_time_minutes: Optional[int] = None + max_total_time_minutes: Optional[int] = None + + +class RetrievalDiagnostics(BaseModel): + query: str + normalized_query: Optional[str] = None + candidate_k: int + returned_k: int + reranked: bool + filters: Optional[RecipeFilters] = None + top_ids: List[str] = Field(default_factory=list) + top_scores: List[float] = Field(default_factory=list) + + class RecipeSearchRequest(BaseModel): query: str = Field(..., description="Ingredients or free-text query") top_k: int = Field(default=5, ge=1, le=50) + filters: Optional[RecipeFilters] = None + rerank: bool = Field(default=True) + include_diagnostics: bool = Field(default=False) + candidate_k: Optional[int] = Field(default=None, ge=1, le=200) + + +class RecommendRequest(BaseModel): + ingredients: List[str] = Field(..., description="Ingredients available") + dietary_notes: Optional[str] = Field(default=None, description="Optional dietary notes") + top_k: int = Field(default=5, ge=1, le=50) + filters: Optional[RecipeFilters] = None + rerank: bool = Field(default=True) + include_diagnostics: bool = Field(default=False) + candidate_k: Optional[int] = Field(default=None, ge=1, le=200) class RecipeSearchResponse(BaseModel): - results: List[Recipe] \ No newline at end of file + results: List[Recipe] + diagnostics: Optional[RetrievalDiagnostics] = None diff --git a/backend/app/schemas/retrieval.py b/backend/app/schemas/retrieval.py new file mode 100644 index 0000000..c374b17 --- /dev/null +++ b/backend/app/schemas/retrieval.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + + +class RetrievalMode(str, Enum): + dense = "dense" + keyword = "keyword" + hybrid = "hybrid" + + +class DocumentType(str, Enum): + recipe = "recipe" + nutrition = "nutrition" + ingredient = "ingredient" + + +class RetrievalFilters(BaseModel): + meal_type: Optional[List[str]] = None + dietary_tags: Optional[List[str]] = None + cuisine: Optional[List[str]] = None + allergens_exclude: Optional[List[str]] = None + + min_calories: Optional[float] = None + max_calories: Optional[float] = None + min_protein_g: Optional[float] = None + max_protein_g: Optional[float] = None + min_carbs_g: Optional[float] = None + max_carbs_g: Optional[float] = None + min_fat_g: Optional[float] = None + max_fat_g: Optional[float] = None + + +class RetrievalRequest(BaseModel): + query: str = Field(..., description="Ingredient list or free-text query") + top_k: int = Field(default=5, ge=1, le=50) + mode: RetrievalMode = Field(default=RetrievalMode.hybrid) + rerank: bool = Field(default=True, description="Apply hybrid reranking when enabled") + doc_types: Optional[List[DocumentType]] = Field(default=None) + filters: Optional[RetrievalFilters] = None + candidate_k: Optional[int] = Field(default=None, ge=1, le=200) + include_diagnostics: bool = Field(default=False) + + +class RetrievedChunk(BaseModel): + chunk_id: str + doc_id: str + doc_type: str + title: Optional[str] = None + text: str + source: Optional[str] = None + score: float + dense_score: Optional[float] = None + keyword_score: Optional[float] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class RetrievalDiagnostics(BaseModel): + mode: RetrievalMode + dense_candidates: Optional[int] = None + keyword_candidates: Optional[int] = None + candidate_k: Optional[int] = None + collection: Optional[str] = None + lexical_index_loaded: Optional[bool] = None + hybrid_alpha: Optional[float] = None + filters: Optional[Dict[str, Any]] = None + + +class RetrievalResponse(BaseModel): + query: str + mode: RetrievalMode + results: List[RetrievedChunk] + diagnostics: Optional[RetrievalDiagnostics] = None diff --git a/backend/app/services/__pycache__/rag_service.cpython-313.pyc b/backend/app/services/__pycache__/rag_service.cpython-313.pyc deleted file mode 100644 index 7afffbf23262a9fa9707a7830d57d302c154eaea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6386 zcmb7IZEPFInV#ix$>oPgTB2lIrmWRhZKjrF$x>`9j(v(LDY9iM=8A4wvZYt#lHQ0~ zGP5h&(%lt7e;lN1ud>>tR?(tx`T_g?Il!THfC3SkCe2SVl8?NNiUw%@1ouZh`z}R; z0)1zfOVTTC3#68d^aU%nUSKeD!G>*|(Z-JLdK(tAdg{PVNa>5N3vTS@P#i^C&7rDk zC*rNcl<|!{2M~NSt%$b`TgqkL3>CD0Og_RvTBAqhMa>$LlB#B%k`y&aX-&K+tD>9~ zXRshE=^UOH@lu1FTfmZXO_2JqkVzV{jM!2Bc$9+7YoHdH8cd7AG;hIHo{lrTHO>rK zi6Spqv5oiete&$&&Y|a6-U+Q-aR+aUJL9f66L$|;7?4MPLpI*6w`$-UuqR^=dNpT6 z$nYY*E+xf}8=;9{fhjx3M<;o{cUDlu6o*9>OX77Q%PB^?K8~BYCUVJ~tf+V)spc>z z3KNP=yNOTHVmC`~|RXFh`EDCotmAfHlb3#gzGh7c0rM3=nWk* zQc~{^YA;|};RG&A2F|6q6zq+ZlLeTK;=CcL*Pt$$o6j$(Fw3h-GKa;hTv`&dsXnLX zOAC^sBqTW{-b~~L^_u3xVp7VBN+Lg()k~5ztd|)`p8OuIL-ZPY(U&!A(2HEQe2fr^ zk!{cePnm}+in0|+Jc2LtxTAjZRp}8$kp{DOH61QzIcOX4R?E#?wLXJ_mhjS-l=7kM zRja2`=@+uu-b<1!O8eD9Oqk<1Eu0*QN8wy4@uYyWLOdiYb80Rh4;zm6) zx0)l%*6%n=!CpP-?1sd>YeSBPB73;N9^PiT66-FqfdU(N$R2(KVvuGtj#)Ta*g1aJ;UidBF=5~#K^bDbzkf5A-u|GQ)7?L2$h z%AriHuI=WS)s%Odt3%pmPzQ?n&00GW02c_o$#gpke=Ciig)~j=X~Q8TTWeON-YAZ! zwYnnhM51-nnklDg%9I>*g=c`%SRKa`xw((YStbRFsH%YG%F>Z*s+w0u`ukG^()%*G zTqY~_0m2#u{pnmb1vkBaX)rQ6!Byabn+Ko;0mnE&RmJ%{5yqJ`VQ@r`G7cDG%Ppt? z)*AEjf+VV%Ye5kc$y{y@;C!hY;5rL14jfVG`ot(BT#vFeAVamzH_^xxCtQa>$jbuIIl)NHrmu;jW2NRIJOZ zBGtZ$P-ODTSYmv7Dl`^JOolYa*v+H}6V1t*4Hyf!Q?p8PI;SxiqP^xd=a4YgOtaq* zunhVI8B8vNA=hD4CZv=`FW{_3i#L-BVG96Oj>Ao)A-trJ7N~{7y=dJ7EfhPz*DGj; zb*{Sp+4bskwNi)uxt=N>K2bP) zqUaqec!z%d+*=cGPP`R@4-?$8O|r zdxWTlL#3A1@6o$7U&?3#SW%2Ti)ziHhc z>OpUN&xTH--}bmeL-cP?P$WL-9P6Opp<2cQ^g96t^7Vng1n~bwz#k!7wI=`o6RzD8 zOcJ~LNMdFnGHmC1}l(_D>&{negk zj;Fs!LJi2)p~%lzsRO3`YWlxc>R|)< zqre)H`=tl4C*aSN1LXVpYs$pB%$}eVXXdME)y}BwdNYt4xvhGr-^Ot zCJ=P`sp?p7*BqZAp|bBfB~{27C<#=RshXLC#(M*AiBi>7(x1DPbEuOKHA+oTVF%C< zXqKc72>AQ3s62DLG|wwe=* z@SxY{iOHeCu(}A7{lH&>NMm#@@RvyedHZSBTwVlnGt|(mWG(Qt*86^2@4Zy9?|h-}{8n4&mi1L-7e=NZ`d)bKL=8== z;g#^_p>rGl;^>va=#_2vS3dge5m*?Z`Jg%OegAgz(+@g!x`KC~`ROzFUjEtHTcKB{ zKA};|$&XB;JGnD(>Vf~Qz?*@M#p1x_!ocMppTA?h>wew69q8Ts%GXK(t{5071P1<_ zvFtkmtYvES+(xJv`f4Hc)ran{{rPd1PH`3(9t6AB+?#D@9wfJY&z0IbZdq?TzvujP zmx0>9eBy(_A2vVzDa?(KVr`Fe8%^83u~Iwf0V2vhlu20I`=iX*AbOi=89#;IJ`@-~ zLce`-^dNk^Gw2-err$Z!Hr`IZ+s;7#-6PKNVf(uScBp;#lyf3T|IX4j(M|uZn}Ixe zu!(q4W5BRW%spW>lh-o=q(|QcP#YATRY>0AX5 zfVvvGF?}x8hTudpVUz<4yqn0YL$LL1@hH__vFgDtsjFc<8$DKmoqDzou|kMNblL*c zQwHA!Nb_U;NVdf)AR7h05P4HX86!2@HlgQVMM&>)Gs^PJ2*6|tY@;X}tH6H!H8a;? z6f~(%ucEdH0k%CMT4QBh`T;^Npp>Ys8wKs*r2sJ@6vNe|pW$bFFqjx{ zik}5*;_u2eN${Nj7tAiGHAam9UMT|T28tgeUNmq=5fg)$IE%VdO0%b-T0JocW)zcB z7xN+}+p96UFDlqrgR(m45&$(YmiU3d?2wb-lF@929-2kYYj!|ajD5k^--vw>p;{3}710m^5|Xy=FoTAZKNsE|fY61R`C zkX6ABN`Y5RqoveMjRu0zD6A0R*G&&2GEjVs2-sJH;)b#jC|3Ro5wSWvtF9H-rhoAM z*fu*_a(P!rRz?c0wmUO-<9Fjb?nVQSzrOAMMyaWJdGbR~(QfTa2 zKELBVxZd?+wdn0Gc)Lsf{kNZAd;a?u)~F95JGC}d^!F6}JzM@Gx2TMQBy9kndw4U6z-k*8U^+0)W^o?&6 zj)y)WRmMm}KG@%OVj;uDG2SPqpir~HT_Gj9&bs#cJcghS2T21)UYO4>;&GC1HXWXc zT8F3M({yA+bxJqDjk~7L>nagLc1EA8#-1^j`z%gF8N4@;at0zW7b)rkk!8&uuxr{EF=p&kRNF|1U&hUHabuxRW>% diff --git a/backend/app/services/__pycache__/recipe_service.cpython-313.pyc b/backend/app/services/__pycache__/recipe_service.cpython-313.pyc deleted file mode 100644 index 4d2ddf88a64fe19e548e103b5e5b18f9affa3796..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1170 zcmY*Y&1(};5TCbSNt3pzty)nJ!4GtcR1Z~ANA7m0Sn3CkU*A2go>}!5Ny|@T@Uv z&Ws6~vuxltwhV{kNkUFVcRu6Ap_hw{>(hGd7``)ckE+vhoS822` zlC>Lm_n~<|SPJ6>&u=z8O3|ZB4^vh|#~y92sJswS93?`iqX)Vl?e1%eq*l-2`_R#s z`8GjN_LEkKE6S$oVo5m|CM}EuJy%!q1kyR55?F^BnC^P^t<(E6I{R~joLw_wbH*Hk zSxXuhK^jAFryn4;9%<%N;l?GHv2L*|?16Egs4PO6;$rIz=?pD`YgSwXhbSHoliH$< zluqTy(8r*L`ZU)@IvW0m=_`At7)tFeYrzsB)t@Daj6NtwesYEUaZIremwe$ny*sTZ(oFnf4o@(<9m>lpUP z=FMH8<)fJ{+t&8%r`|oilO3ebsJJ=hBrDA*uBMF;r9mxR_T^~CY1)`7(ihWxq1g;+ p{ewNJ{CGyg1fHXl^=B@Rk=ivGV?SZ|C!D6=p5w5Q-#{vD;vb9N5JCU| diff --git a/backend/app/services/embedding_service.py b/backend/app/services/embedding_service.py new file mode 100644 index 0000000..f82cf3a --- /dev/null +++ b/backend/app/services/embedding_service.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import logging +from typing import List + +from sentence_transformers import SentenceTransformer + +logger = logging.getLogger("nourishai.embeddings") + + +class EmbeddingService: + def __init__(self, model_name: str): + self.model_name = model_name + self._model: SentenceTransformer | None = None + + def embed(self, texts: List[str]) -> List[List[float]]: + if not texts: + return [] + if self._model is None: + logger.info("Loading embedding model: %s", self.model_name) + self._model = SentenceTransformer(self.model_name) + vectors = self._model.encode(texts, show_progress_bar=False, normalize_embeddings=True) + return vectors.tolist() diff --git a/backend/app/services/filters.py b/backend/app/services/filters.py new file mode 100644 index 0000000..a28a404 --- /dev/null +++ b/backend/app/services/filters.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from typing import Iterable, List, Optional + +from qdrant_client.models import FieldCondition, Filter, MatchAny, Range + +from ..schemas.retrieval import RetrievalFilters + + +def build_qdrant_filter( + filters: Optional[RetrievalFilters], + doc_types: Optional[List[str]] = None, +) -> Optional[Filter]: + if not filters and not doc_types: + return None + + must: List[FieldCondition] = [] + must_not: List[FieldCondition] = [] + + doc_types_norm = _normalize_list(doc_types) + if doc_types_norm: + must.append(FieldCondition(key="doc_type", match=MatchAny(any=doc_types_norm))) + + if filters: + meal_type = _normalize_list(filters.meal_type) + if meal_type: + must.append(FieldCondition(key="meal_type", match=MatchAny(any=meal_type))) + + cuisine = _normalize_list(filters.cuisine) + if cuisine: + must.append(FieldCondition(key="cuisine", match=MatchAny(any=cuisine))) + + dietary_tags = _normalize_list(filters.dietary_tags) + if dietary_tags: + must.append(FieldCondition(key="dietary_tags", match=MatchAny(any=dietary_tags))) + + allergens_exclude = _normalize_list(filters.allergens_exclude) + if allergens_exclude: + must_not.append(FieldCondition(key="allergens", match=MatchAny(any=allergens_exclude))) + + _add_range(must, "calories", filters.min_calories, filters.max_calories) + _add_range(must, "protein_g", filters.min_protein_g, filters.max_protein_g) + _add_range(must, "carbs_g", filters.min_carbs_g, filters.max_carbs_g) + _add_range(must, "fat_g", filters.min_fat_g, filters.max_fat_g) + + if not must and not must_not: + return None + + return Filter(must=must or None, must_not=must_not or None) + + +def payload_matches(payload: dict, filters: Optional[RetrievalFilters], doc_types: Optional[List[str]] = None) -> bool: + if doc_types: + if str(payload.get("doc_type") or "").lower() not in set(_normalize_list(doc_types)): + return False + + if not filters: + return True + + if filters.meal_type and not _list_matches(payload.get("meal_type"), filters.meal_type): + return False + if filters.cuisine and not _list_matches(payload.get("cuisine"), filters.cuisine): + return False + if filters.dietary_tags and not _list_matches(payload.get("dietary_tags"), filters.dietary_tags): + return False + if filters.allergens_exclude and _list_matches(payload.get("allergens"), filters.allergens_exclude): + return False + + if not _range_matches(payload.get("calories"), filters.min_calories, filters.max_calories): + return False + if not _range_matches(payload.get("protein_g"), filters.min_protein_g, filters.max_protein_g): + return False + if not _range_matches(payload.get("carbs_g"), filters.min_carbs_g, filters.max_carbs_g): + return False + if not _range_matches(payload.get("fat_g"), filters.min_fat_g, filters.max_fat_g): + return False + + return True + + +def _add_range(conditions: List[FieldCondition], key: str, min_value, max_value) -> None: + if min_value is None and max_value is None: + return + conditions.append(FieldCondition(key=key, range=Range(gte=min_value, lte=max_value))) + + +def _list_matches(payload_value, filter_values: Iterable[str]) -> bool: + payload_list = _normalize_list(payload_value) + if not payload_list: + return False + filter_list = set(_normalize_list(filter_values)) + return bool(filter_list.intersection(payload_list)) + + +def _range_matches(payload_value, min_value, max_value) -> bool: + if payload_value is None: + return min_value is None and max_value is None + try: + value = float(payload_value) + except (TypeError, ValueError): + return False + if min_value is not None and value < min_value: + return False + if max_value is not None and value > max_value: + return False + return True + + +def _normalize_list(value) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v).lower() for v in value if str(v).strip()] + if isinstance(value, str): + if "," in value: + return [v.strip().lower() for v in value.split(",") if v.strip()] + return [value.strip().lower()] + return [str(value).lower()] diff --git a/backend/app/services/grounded_ai_service.py b/backend/app/services/grounded_ai_service.py new file mode 100644 index 0000000..09b7e17 --- /dev/null +++ b/backend/app/services/grounded_ai_service.py @@ -0,0 +1,450 @@ +from __future__ import annotations + +import json +import logging +import re +import time +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from ..schemas.ai import ( + GroundedRecommendationRequest, + GroundedRecommendationResponse, + LLMMealPlanPayload, + LLMNutritionQAPayload, + LLMRecommendationPayload, + MacroSummary, + MealPlanRequest, + MealPlanResponse, + NutritionQARequest, + NutritionQAResponse, + RecommendedRecipe, + SourceRecipeRef, +) +from ..schemas.recipe import RecipeFilters, RetrievalDiagnostics +from ..core.observability import log_event +from .llm_service import LLMService +from .rag_service import RagService + +logger = logging.getLogger("nourishai.grounded") + +KNOWN_ALLERGENS = { + "gluten", + "dairy", + "soy", + "egg", + "fish", + "shellfish", + "peanut", + "peanuts", + "tree nuts", + "sesame", +} + +DIET_TAG_TO_ALLERGEN = { + "gluten_free": "gluten", + "dairy_free": "dairy", + "soy_free": "soy", + "egg_free": "egg", + "nut_free": "tree nuts", + "peanut_free": "peanut", + "sesame_free": "sesame", +} + + +class GroundedAIService: + def __init__(self, rag: RagService, llm: LLMService) -> None: + self.rag = rag + self.llm = llm + + def grounded_recommendations(self, request: GroundedRecommendationRequest) -> GroundedRecommendationResponse: + query = f"{', '.join(request.ingredients)}. {request.dietary_notes or ''}".strip() + context_k = request.candidate_k or max(request.top_k * 3, request.top_k + 5) + results, diagnostics = self.rag.search_with_diagnostics( + query=query, + top_k=context_k, + filters=request.filters, + rerank=request.rerank, + candidate_k=context_k, + ingredients=request.ingredients, + include_diagnostics=request.include_diagnostics, + ) + + if not results: + log_event( + logger, + "grounded_recommendations_fallback", + reason="no_results", + query=query, + filters=request.filters.model_dump() if request.filters else None, + ) + return GroundedRecommendationResponse( + recommended_recipes=[], + warnings=["No recipes matched the query and filters."], + source_recipes=[], + retrieval_diagnostics=diagnostics, + ) + + context = self._build_recipe_context(results) + system_prompt = ( + "You are NourishAI. Use ONLY the provided recipes as sources. " + "Do not invent recipes, ingredients, or nutrition facts. " + "If the context is weak, return fewer items and add a warning." + ) + user_prompt = ( + "Task: Recommend recipes based on the user's ingredients and dietary notes. " + "Return JSON that matches the schema. " + "Include why each recipe was selected, substitutions for missing ingredients, " + "and warnings for dietary conflicts if present.\n\n" + f"User ingredients: {request.ingredients}\n" + f"Dietary notes: {request.dietary_notes or 'none'}\n\n" + f"Recipes:\n{context}" + ) + + log_event( + logger, + "grounded_recommendations_context", + source_ids=[str(r.get("id", "")) for r in results], + source_count=len(results), + ) + + llm_start = time.perf_counter() + try: + payload = self.llm.generate_structured(system_prompt, user_prompt, LLMRecommendationPayload) + except Exception as exc: + log_event( + logger, + "grounded_recommendations_llm_failed", + error=str(exc), + source_ids=[str(r.get("id", "")) for r in results], + ) + return GroundedRecommendationResponse( + recommended_recipes=[], + warnings=["LLM generation failed; no grounded recommendations were returned."], + source_recipes=self._source_refs(results), + retrieval_diagnostics=diagnostics, + ) + llm_ms = (time.perf_counter() - llm_start) * 1000.0 + + allowed_ids = {str(r.get("id", "")) for r in results} + recipe_map = {str(r.get("id", "")): r for r in results} + filtered = [r for r in payload.recommended_recipes if r.recipe_id in allowed_ids] + filtered = filtered[: request.top_k] + dropped = len(payload.recommended_recipes) - len(filtered) + + warnings = list(payload.warnings) + if dropped: + warnings.append("Some recommendations were dropped because they were not in retrieved sources.") + + filtered, conflict_warnings = self._collect_conflicts(filtered, recipe_map, request.filters, request.dietary_notes) + warnings.extend(conflict_warnings) + + source_refs = self._source_refs(results) + + log_event( + logger, + "grounded_recommendations_complete", + source_count=len(results), + selected_ids=[r.recipe_id for r in filtered], + dropped_count=dropped, + warning_count=len(warnings), + llm_latency_ms=round(llm_ms, 2), + ) + + return GroundedRecommendationResponse( + recommended_recipes=filtered, + warnings=warnings, + source_recipes=source_refs, + retrieval_diagnostics=diagnostics, + ) + + def generate_meal_plan(self, request: MealPlanRequest) -> MealPlanResponse: + query_parts = [request.dietary_profile or "", "meal plan", "balanced"] + if request.meal_preferences: + query_parts.append(", ".join(request.meal_preferences)) + query = ". ".join(part for part in query_parts if part).strip() + + candidate_k = request.candidate_k or max(request.days * request.meals_per_day * 3, 10) + results, diagnostics = self.rag.search_with_diagnostics( + query=query, + top_k=candidate_k, + filters=request.filters, + rerank=request.rerank, + candidate_k=candidate_k, + include_diagnostics=request.include_diagnostics, + ) + + if not results: + log_event( + logger, + "meal_plan_fallback", + reason="no_results", + query=query, + filters=request.filters.model_dump() if request.filters else None, + ) + return MealPlanResponse( + days=[], + dietary_notes=request.dietary_profile, + warnings=["No recipes matched the meal plan query."], + source_recipes=[], + retrieval_diagnostics=diagnostics, + ) + + context = self._build_recipe_context(results) + system_prompt = ( + "You are NourishAI. Build a meal plan using ONLY the provided recipes. " + "Do not invent recipes or nutrition values. If data is missing, leave fields null." + ) + user_prompt = ( + "Task: Create a multi-day meal plan using the provided recipes. " + "Return JSON that matches the schema.\n\n" + f"Dietary profile: {request.dietary_profile or 'none'}\n" + f"Calorie target per day: {request.calorie_target or 'unspecified'}\n" + f"Days: {request.days}\n" + f"Meals per day: {request.meals_per_day}\n" + f"Meal preferences: {request.meal_preferences or []}\n\n" + f"Recipes:\n{context}" + ) + + log_event( + logger, + "meal_plan_context", + source_ids=[str(r.get("id", "")) for r in results], + source_count=len(results), + ) + + llm_start = time.perf_counter() + try: + payload = self.llm.generate_structured(system_prompt, user_prompt, LLMMealPlanPayload) + except Exception as exc: + log_event( + logger, + "meal_plan_llm_failed", + error=str(exc), + source_ids=[str(r.get("id", "")) for r in results], + ) + return MealPlanResponse( + days=[], + dietary_notes=request.dietary_profile, + warnings=["LLM generation failed; no meal plan was returned."], + source_recipes=self._source_refs(results), + retrieval_diagnostics=diagnostics, + ) + llm_ms = (time.perf_counter() - llm_start) * 1000.0 + allowed_ids = {str(r.get("id", "")) for r in results} + + filtered_days = [] + removed = 0 + for day in payload.days: + meals = [meal for meal in day.meals if meal.recipe_id in allowed_ids] + removed += len(day.meals) - len(meals) + filtered_days.append(day.model_copy(update={"meals": meals})) + + warnings = list(payload.warnings) + if removed: + warnings.append("Some meals were removed because they were not in retrieved sources.") + + source_refs = self._source_refs(results) + meal_ids = [meal.recipe_id for day in filtered_days for meal in day.meals] + log_event( + logger, + "meal_plan_complete", + source_count=len(results), + selected_ids=meal_ids, + removed_count=removed, + warning_count=len(warnings), + llm_latency_ms=round(llm_ms, 2), + ) + + return MealPlanResponse( + days=filtered_days, + dietary_notes=payload.dietary_notes or request.dietary_profile, + warnings=warnings, + source_recipes=source_refs, + retrieval_diagnostics=diagnostics, + ) + + def answer_nutrition_question(self, request: NutritionQARequest) -> NutritionQAResponse: + results, diagnostics = self.rag.search_with_diagnostics( + query=request.question, + top_k=request.top_k, + filters=request.filters, + rerank=request.rerank, + candidate_k=request.candidate_k, + include_diagnostics=request.include_diagnostics, + ) + + if not results: + log_event( + logger, + "nutrition_qa_fallback", + reason="no_results", + query=request.question, + filters=request.filters.model_dump() if request.filters else None, + ) + return NutritionQAResponse( + answer="I could not find supporting recipes for that question.", + supporting_sources=[], + warnings=["No relevant recipes found."], + retrieval_diagnostics=diagnostics, + ) + + context = self._build_recipe_context(results) + system_prompt = ( + "You are NourishAI. Answer ONLY using the provided recipes. " + "Do not invent nutrition facts. If uncertain, say so." + ) + user_prompt = ( + "Task: Answer the nutrition question using the recipes below. " + "Return JSON that matches the schema.\n\n" + f"Question: {request.question}\n\n" + f"Recipes:\n{context}" + ) + + log_event( + logger, + "nutrition_qa_context", + source_ids=[str(r.get("id", "")) for r in results], + source_count=len(results), + ) + + llm_start = time.perf_counter() + try: + payload = self.llm.generate_structured(system_prompt, user_prompt, LLMNutritionQAPayload) + except Exception as exc: + log_event( + logger, + "nutrition_qa_llm_failed", + error=str(exc), + source_ids=[str(r.get("id", "")) for r in results], + ) + return NutritionQAResponse( + answer="LLM generation failed; unable to answer using retrieved sources.", + supporting_sources=self._source_refs(results), + warnings=["LLM generation failed; returning sources only."], + retrieval_diagnostics=diagnostics, + ) + llm_ms = (time.perf_counter() - llm_start) * 1000.0 + allowed_ids = {str(r.get("id", "")) for r in results} + supporting = [rid for rid in payload.supporting_recipe_ids if rid in allowed_ids] + if not supporting: + supporting = [str(r.get("id", "")) for r in results[: min(3, len(results))]] + payload = payload.model_copy(update={"warnings": payload.warnings + ["No supporting source IDs were provided; defaults were used."]}) + + source_refs = self._source_refs(results) + supporting_refs = [ref for ref in source_refs if ref.id in supporting] + + log_event( + logger, + "nutrition_qa_complete", + source_count=len(results), + supporting_ids=supporting, + warning_count=len(payload.warnings), + llm_latency_ms=round(llm_ms, 2), + ) + + return NutritionQAResponse( + answer=payload.answer, + supporting_sources=supporting_refs, + warnings=payload.warnings, + retrieval_diagnostics=diagnostics, + ) + + def _build_recipe_context(self, recipes: Sequence[Dict[str, Any]]) -> str: + context_payload = [] + for recipe in recipes: + context_payload.append( + { + "id": str(recipe.get("id", "")), + "title": recipe.get("title", ""), + "ingredients": recipe.get("ingredients") or [], + "instructions": recipe.get("instructions") or [], + "calories": recipe.get("calories"), + "protein_g": recipe.get("protein_g"), + "carbs_g": recipe.get("carbs_g"), + "fat_g": recipe.get("fat_g"), + "cuisine": recipe.get("cuisine"), + "meal_type": recipe.get("meal_type"), + "dietary_tags": recipe.get("dietary_tags"), + "allergens": recipe.get("allergens"), + "prep_time_minutes": recipe.get("prep_time_minutes"), + "cook_time_minutes": recipe.get("cook_time_minutes"), + "total_time_minutes": recipe.get("total_time_minutes"), + } + ) + return json.dumps(context_payload, ensure_ascii=True) + + @staticmethod + def _source_refs(recipes: Sequence[Dict[str, Any]]) -> List[SourceRecipeRef]: + refs = [] + for recipe in recipes: + refs.append(SourceRecipeRef(id=str(recipe.get("id", "")), title=recipe.get("title", ""))) + return refs + + def _collect_conflicts( + self, + recommended: Sequence[RecommendedRecipe], + recipe_map: Dict[str, Dict[str, Any]], + filters: Optional[RecipeFilters], + dietary_notes: Optional[str], + ) -> Tuple[List[RecommendedRecipe], List[str]]: + warnings: List[str] = [] + restricted = self._restricted_allergens(filters, dietary_notes) + if not restricted: + return list(recommended), warnings + + restricted_set = {r.lower() for r in restricted} + updated: List[RecommendedRecipe] = [] + + for item in recommended: + recipe = recipe_map.get(item.recipe_id, {}) + allergens = _normalize_list(recipe.get("allergens")) + conflicts = restricted_set.intersection(allergens) + if conflicts: + extra = [f"Contains {allergen}." for allergen in sorted(conflicts)] + merged = list(dict.fromkeys(item.warnings + extra)) + updated.append(item.model_copy(update={"warnings": merged})) + else: + updated.append(item) + + if restricted_set: + warnings.append( + "Dietary restrictions were detected; verify allergens in the recommended recipes." + ) + return updated, warnings + + def _restricted_allergens(self, filters: Optional[RecipeFilters], dietary_notes: Optional[str]) -> List[str]: + restricted: List[str] = [] + if filters and filters.dietary_tags: + for tag in filters.dietary_tags: + key = str(tag).lower() + if key in DIET_TAG_TO_ALLERGEN: + restricted.append(DIET_TAG_TO_ALLERGEN[key]) + if dietary_notes: + restricted.extend(_extract_allergens_from_notes(dietary_notes)) + return list({r for r in restricted if r}) + + +def _extract_allergens_from_notes(text: str) -> List[str]: + notes = text.lower() + allergens: List[str] = [] + for allergen in KNOWN_ALLERGENS: + if allergen in notes: + allergens.append(allergen) + match = re.findall(r"no\s+([a-z\s]+)", notes) + for item in match: + value = item.strip() + if value: + allergens.append(value) + return allergens + + +def _normalize_list(value: Any) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v).lower() for v in value if str(v).strip()] + if isinstance(value, str): + if "," in value: + return [v.strip().lower() for v in value.split(",") if v.strip()] + return [value.strip().lower()] + return [str(value).lower()] diff --git a/backend/app/services/ingredient_normalizer.py b/backend/app/services/ingredient_normalizer.py new file mode 100644 index 0000000..949b9c2 --- /dev/null +++ b/backend/app/services/ingredient_normalizer.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import re +from functools import lru_cache +from typing import Iterable, List + +ALIAS_MAP = { + "garbanzo bean": "chickpea", + "garbanzo beans": "chickpea", + "chick peas": "chickpea", + "scallion": "green onion", + "scallions": "green onion", + "spring onion": "green onion", + "spring onions": "green onion", + "bell peppers": "bell pepper", + "courgette": "zucchini", + "capsicum": "bell pepper", + "cilantro": "coriander", + "ground beef": "beef", +} + + +@lru_cache(maxsize=2048) +def normalize_ingredient(text: str) -> str: + if not text: + return "" + cleaned = text.lower().strip() + cleaned = re.sub(r"\(.*?\)", "", cleaned) + cleaned = re.sub(r"[^a-z0-9\s-]", "", cleaned) + cleaned = cleaned.replace("-", " ") + cleaned = re.sub(r"\s+", " ", cleaned).strip() + + cleaned = _singularize(cleaned) + cleaned = ALIAS_MAP.get(cleaned, cleaned) + return cleaned + + +def normalize_ingredient_list(items: Iterable[str]) -> List[str]: + normalized: List[str] = [] + for item in items: + value = normalize_ingredient(str(item)) + if value: + normalized.append(value) + return normalized + + +def extract_ingredients_from_query(query: str) -> List[str]: + if not query: + return [] + if "," in query: + parts = [p.strip() for p in query.split(",") if p.strip()] + if len(parts) >= 2: + return normalize_ingredient_list(parts) + tokens = [t.strip() for t in re.split(r"\band\b", query, flags=re.IGNORECASE) if t.strip()] + if len(tokens) >= 2: + return normalize_ingredient_list(tokens) + return [] + + +def _singularize(text: str) -> str: + if text.endswith("ies") and len(text) > 3: + return text[:-3] + "y" + if text.endswith("es") and len(text) > 3: + return text[:-2] + if text.endswith("s") and len(text) > 3: + return text[:-1] + return text diff --git a/backend/app/services/llm_service.py b/backend/app/services/llm_service.py new file mode 100644 index 0000000..bba3ee8 --- /dev/null +++ b/backend/app/services/llm_service.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Optional, Type + +from openai import OpenAI +from openai import OpenAIError +from pydantic import BaseModel, ValidationError + +from ..core.config import settings +from ..core.observability import log_event + +logger = logging.getLogger("nourishai.llm") + + +class LLMService: + def __init__( + self, + api_key: Optional[str] = None, + model: Optional[str] = None, + temperature: Optional[float] = None, + max_tokens: Optional[int] = None, + timeout_seconds: Optional[float] = None, + client: Optional[OpenAI] = None, + ) -> None: + self.api_key = api_key or settings.OPENAI_API_KEY + self.model = model or settings.OPENAI_MODEL + self.temperature = temperature if temperature is not None else settings.OPENAI_TEMPERATURE + self.max_tokens = max_tokens if max_tokens is not None else settings.OPENAI_MAX_TOKENS + self.timeout_seconds = timeout_seconds if timeout_seconds is not None else settings.OPENAI_TIMEOUT_SEC + + if not self.api_key: + logger.error("OPENAI_API_KEY is not set. LLM features are disabled.") + self._client: Optional[OpenAI] = None + else: + self._client = client or OpenAI(api_key=self.api_key, timeout=self.timeout_seconds) + + def is_configured(self) -> bool: + return self._client is not None + + def generate_structured(self, system_prompt: str, user_prompt: str, schema_model: Type[BaseModel]) -> BaseModel: + if self._client is None: + raise RuntimeError("OpenAI API key is missing. Set OPENAI_API_KEY to enable LLM features.") + + schema = schema_model.model_json_schema() + response_format = { + "type": "json_schema", + "json_schema": { + "name": schema_model.__name__, + "schema": schema, + "strict": True, + }, + } + + start = time.perf_counter() + success = False + try: + response = self._client.chat.completions.create( + model=self.model, + temperature=self.temperature, + max_tokens=self.max_tokens, + response_format=response_format, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + ) + success = True + except OpenAIError as exc: + log_event( + logger, + "llm_error", + model=self.model, + error=str(exc), + ) + raise RuntimeError("OpenAI request failed") from exc + finally: + latency_ms = (time.perf_counter() - start) * 1000.0 + log_event( + logger, + "llm_call", + model=self.model, + schema=schema_model.__name__, + success=success, + latency_ms=round(latency_ms, 2), + ) + + content = response.choices[0].message.content if response.choices else None + if not content: + raise RuntimeError("OpenAI response was empty") + + try: + payload: Any = json.loads(content) + except json.JSONDecodeError as exc: + log_event(logger, "llm_invalid_json", error=str(exc)) + raise RuntimeError("OpenAI response was not valid JSON") from exc + + try: + parsed = schema_model.model_validate(payload) + except ValidationError as exc: + log_event(logger, "llm_schema_validation_failed", error=str(exc)) + raise RuntimeError("OpenAI response failed schema validation") from exc + + log_event(logger, "llm_response_validated", schema=schema_model.__name__) + return parsed diff --git a/backend/app/services/model_store.py b/backend/app/services/model_store.py index 7edb35a..6191000 100644 --- a/backend/app/services/model_store.py +++ b/backend/app/services/model_store.py @@ -6,7 +6,7 @@ import numpy as np import requests -from app.core.config import settings +from ..core.config import settings logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def __init__(self): self.category_dict: Dict[int, str] = {} self.recipe_dict: Dict[str, Any] = {} self.faiss_index: Any = None - self.rag_texts: List[str] = [] + self.rag_texts: List[dict] = [] self.sentence_model: Any = None async def load_all(self): @@ -30,17 +30,17 @@ async def load_all(self): self._try_load_rag() def _load_category_dict(self): - path = settings.category_dict_path + path = settings.CATEGORY_DICT_PATH if not os.path.exists(path): - logger.warning("category_dict.npy not found at %s – image prediction disabled.", path) + logger.warning("category_dict.npy not found at %s - image prediction disabled.", path) return self.category_dict = np.load(path, allow_pickle=True).item() logger.info("Loaded category_dict with %d classes.", len(self.category_dict)) def _load_recipe_dict(self): - path = settings.raw_recipes_path + path = settings.RAW_RECIPES_PATH if not os.path.exists(path): - logger.warning("raw_recipes.npy not found – downloading from Google Drive...") + logger.warning("raw_recipes.npy not found - downloading from Google Drive...") self._download_gdrive(settings.GDRIVE_RAW_RECIPES_ID, path) if os.path.exists(path): self.recipe_dict = np.load(path, allow_pickle=True).item() @@ -49,7 +49,7 @@ def _load_recipe_dict(self): logger.error("Could not load recipe_dict.") def _load_image_model(self): - path = settings.image_model_path + path = settings.IMAGE_MODEL_PATH expected = settings.IMAGE_MODEL_EXPECTED_SIZE_MB * 1024 * 1024 if not os.path.exists(path) or os.path.getsize(path) < expected: logger.info("Downloading image model from GitHub Releases...") @@ -62,19 +62,20 @@ def _load_image_model(self): logger.error("Image model not available.") def _try_load_rag(self): - faiss_path = settings.faiss_index_path - data_path = settings.recipes_data_path + faiss_path = settings.FAISS_INDEX_PATH + data_path = settings.RECIPES_PKL_PATH if not (os.path.exists(faiss_path) and os.path.exists(data_path)): - logger.info("FAISS index not found – RAG endpoint will be unavailable.") + logger.info("FAISS index not found - RAG endpoint will be unavailable.") return try: import faiss from sentence_transformers import SentenceTransformer + self.faiss_index = faiss.read_index(faiss_path) with open(data_path, "rb") as f: self.rag_texts = pickle.load(f) self.sentence_model = SentenceTransformer("all-MiniLM-L6-v2") - logger.info("RAG components loaded (%d texts).", len(self.rag_texts)) + logger.info("RAG components loaded (%d recipes).", len(self.rag_texts)) except Exception as exc: logger.warning("Could not load RAG components: %s", exc) @@ -83,6 +84,7 @@ def _download_gdrive(file_id: str, destination: str): url = f"https://drive.google.com/uc?id={file_id}" try: import gdown + gdown.download(url, destination, quiet=False) except Exception as exc: logger.error("gdown failed: %s", exc) @@ -96,7 +98,7 @@ def _download_stream(url: str, destination: str): for chunk in r.iter_content(chunk_size=8192): if chunk: f.write(chunk) - logger.info("Downloaded %s → %s", url, destination) + logger.info("Downloaded %s -> %s", url, destination) except Exception as exc: logger.error("Stream download failed: %s", exc) diff --git a/backend/app/services/rag_service.py b/backend/app/services/rag_service.py index e985c32..5935511 100644 --- a/backend/app/services/rag_service.py +++ b/backend/app/services/rag_service.py @@ -1,111 +1,383 @@ +from __future__ import annotations + import logging import os import pickle -from typing import Any, Dict, List +import time +from typing import Any, Dict, List, Optional, Tuple import faiss -import gdown import numpy as np from sentence_transformers import SentenceTransformer +from starlette.exceptions import HTTPException -logger = logging.getLogger("nourishai.rag") - - - -# we'll construct the download URL from settings when needed +from .ingredient_normalizer import extract_ingredients_from_query, normalize_ingredient_list +from ..core.observability import log_event +from ..schemas.recipe import RecipeFilters, RetrievalDiagnostics class RagService: - """ - FAISS-based retrieval service. The constructor eagerly ensures that the - artifacts exist (downloading from Drive if necessary) and then loads them - into memory. Search is pure business logic and returns a list of dictionaries - with a computed `score` field. - """ + logger = logging.getLogger("nourishai.rag") def __init__(self, faiss_index_path: str, recipes_pkl_path: str): self.faiss_index_path = faiss_index_path self.recipes_pkl_path = recipes_pkl_path - # transformer model is expensive to download; load lazily in search - self._model: SentenceTransformer | None = None self.index: faiss.Index | None = None - self.recipes: List[Any] = [] + self.recipes: List[Dict[str, Any]] = [] + self._model: SentenceTransformer | None = None - # ensure the FAISS/recipe artifacts are on disk (download if necessary) - self._ensure_artifacts() self._load_artifacts() - def _ensure_artifacts(self) -> None: - os.makedirs(os.path.dirname(self.faiss_index_path), exist_ok=True) + def search( + self, + query: str, + top_k: int, + filters: Optional[RecipeFilters] = None, + rerank: bool = True, + candidate_k: Optional[int] = None, + ingredients: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + results, _ = self.search_with_diagnostics( + query=query, + top_k=top_k, + filters=filters, + rerank=rerank, + candidate_k=candidate_k, + ingredients=ingredients, + include_diagnostics=False, + ) + return results + + def search_with_diagnostics( + self, + query: str, + top_k: int, + filters: Optional[RecipeFilters] = None, + rerank: bool = True, + candidate_k: Optional[int] = None, + ingredients: Optional[List[str]] = None, + include_diagnostics: bool = False, + ) -> Tuple[List[Dict[str, Any]], Optional[RetrievalDiagnostics]]: + if self.index is None or not self.recipes: + log_event( + self.logger, + "rag_unavailable", + index_loaded=self.index is not None, + recipes_loaded=bool(self.recipes), + ) + raise HTTPException(status_code=503, detail="RAG index not available. Build it with scripts/build_rag_index.py") + + if not query.strip() or top_k <= 0: + diagnostics = self._build_diagnostics( + query=query, + normalized_query=query.strip(), + candidate_k=0, + returned_k=0, + filters=filters, + reranked=False, + top_ids=[], + top_scores=[], + ) if include_diagnostics else None + log_event( + self.logger, + "rag_empty_query", + query=query, + top_k=top_k, + ) + return [], diagnostics - if not os.path.exists(self.faiss_index_path) or not os.path.exists(self.recipes_pkl_path): - # fetch the ID from settings if available - try: - from ..core.config import settings - folder_id = settings.GDRIVE_FOLDER_ID - url = f"https://drive.google.com/drive/folders/{folder_id}" - except Exception: # settings might not be initialised yet - url = None - - if url: - logger.info("RAG artifacts missing, attempting download from %s", url) - try: - gdown.download_folder(url, output=os.path.dirname(self.faiss_index_path), quiet=False, use_cookies=False) - except Exception as exc: # pragma: no cover - network may be unavailable - logger.warning("failed to download RAG artifacts: %s", exc) + start = time.perf_counter() + normalized_ingredients = normalize_ingredient_list(ingredients or []) + if not normalized_ingredients: + normalized_ingredients = extract_ingredients_from_query(query) + + normalized_query = ", ".join(normalized_ingredients) if normalized_ingredients else query.strip() + + if candidate_k is None: + if filters or normalized_ingredients or rerank: + candidate_k = max(top_k * 5, top_k + 10) else: - logger.warning("no drive URL available, skipping download attempt") + candidate_k = top_k + candidate_k = max(candidate_k, top_k) + + if self._model is None: + self._model = SentenceTransformer("all-MiniLM-L6-v2") + + embed_start = time.perf_counter() + embedding = self._model.encode([normalized_query], show_progress_bar=False) + query_vector = np.asarray(embedding, dtype="float32") + faiss.normalize_L2(query_vector) + embed_ms = (time.perf_counter() - embed_start) * 1000.0 + + search_start = time.perf_counter() + scores, indices = self.index.search(query_vector, candidate_k) + search_ms = (time.perf_counter() - search_start) * 1000.0 + + filter_start = time.perf_counter() + candidates: List[Tuple[Dict[str, Any], float]] = [] + for score, idx in zip(scores[0], indices[0]): + if idx < 0 or idx >= len(self.recipes): + continue + recipe = self.recipes[idx] + if not self._matches_filters(recipe, filters): + continue + candidates.append((recipe, float(score))) + filter_ms = (time.perf_counter() - filter_start) * 1000.0 + + reranked = False + rerank_start = time.perf_counter() + if rerank and normalized_ingredients: + reranked = True + candidates = self._rerank_candidates(candidates, normalized_ingredients) + else: + candidates.sort(key=lambda item: item[1], reverse=True) + rerank_ms = (time.perf_counter() - rerank_start) * 1000.0 + + final = candidates[:top_k] + results = [self._recipe_to_result(recipe, score) for recipe, score in final] + + diagnostics = None + if include_diagnostics: + top_ids = [str(r.get("id", "")) for r, _ in final] + top_scores = [float(score) for _, score in final] + diagnostics = self._build_diagnostics( + query=query, + normalized_query=normalized_query, + candidate_k=candidate_k, + returned_k=len(results), + filters=filters, + reranked=reranked, + top_ids=top_ids, + top_scores=top_scores, + ) + + total_ms = (time.perf_counter() - start) * 1000.0 + strategy = "reranked" if reranked else ("filtered" if filters else "baseline") + log_event( + self.logger, + "rag_search", + query=query, + normalized_query=normalized_query, + top_k=top_k, + candidate_k=candidate_k, + strategy=strategy, + reranked=reranked, + filters=filters.model_dump() if filters else None, + result_ids=[r.get("id") for r in results], + result_scores=[r.get("score") for r in results], + timings_ms={ + "total": round(total_ms, 2), + "embed": round(embed_ms, 2), + "search": round(search_ms, 2), + "filter": round(filter_ms, 2), + "rerank": round(rerank_ms, 2), + }, + ) + + return results, diagnostics def _load_artifacts(self) -> None: if os.path.exists(self.faiss_index_path): try: self.index = faiss.read_index(self.faiss_index_path) - logger.info("loaded faiss index from %s", self.faiss_index_path) + self.logger.info("Loaded FAISS index from %s", self.faiss_index_path) except Exception as exc: - logger.error("error loading faiss index: %s", exc) + self.logger.warning("Failed to load FAISS index: %s", exc) self.index = None else: - logger.warning("faiss index path %s does not exist", self.faiss_index_path) + self.logger.warning("FAISS index path %s does not exist", self.faiss_index_path) if os.path.exists(self.recipes_pkl_path): try: - with open(self.recipes_pkl_path, "rb") as f: - self.recipes = pickle.load(f) - logger.info("loaded %s recipes from %s", - len(self.recipes) if hasattr(self.recipes, "__len__") else "?", - self.recipes_pkl_path) + with open(self.recipes_pkl_path, "rb") as handle: + data = pickle.load(handle) + if isinstance(data, list): + self.recipes = data + else: + self.logger.warning("recipes.pkl did not contain a list; got %s", type(data)) + self.recipes = [] + self.logger.info("Loaded %d recipes from %s", len(self.recipes), self.recipes_pkl_path) except Exception as exc: - logger.error("error loading recipes pickle: %s", exc) + self.logger.warning("Failed to load recipes pickle: %s", exc) self.recipes = [] else: - logger.warning("recipes pickle path %s does not exist", self.recipes_pkl_path) + self.logger.warning("recipes pickle path %s does not exist", self.recipes_pkl_path) - def search(self, query: str, top_k: int) -> List[Dict[str, Any]]: - logger.info("RAG search query=%r top_k=%s", query, top_k) - if self.index is None or not self.recipes: - logger.warning("search called but artifacts are not loaded, returning empty list") - return [] + if self.index is not None and self.recipes: + index_count = int(self.index.ntotal) + if index_count != len(self.recipes): + self.logger.warning( + "FAISS index count (%s) does not match recipes count (%s)", + index_count, + len(self.recipes), + ) - # load the encoder only when we actually need it - if self._model is None: - try: - self._model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") - except Exception as exc: # network or model error - logger.error("failed to load sentence-transformer model: %s", exc) - return [] - - emb = self._model.encode([query]) - # faiss expects float32 - D, I = self.index.search(np.asarray(emb, dtype="float32"), top_k) - results: List[Dict[str, Any]] = [] - for dist, idx in zip(D[0], I[0]): - if idx < 0 or idx >= len(self.recipes): - continue - rec = self.recipes[idx] - if isinstance(rec, dict): - r = rec.copy() - else: - r = rec.__dict__.copy() - r["score"] = float(dist) - results.append(r) - return results + def status(self) -> Dict[str, Any]: + index_loaded = self.index is not None + recipes_loaded = bool(self.recipes) + index_count = int(self.index.ntotal) if self.index is not None else 0 + recipe_count = len(self.recipes) + artifact_mismatch = index_loaded and recipes_loaded and index_count != recipe_count + return { + "index_loaded": index_loaded, + "recipes_loaded": recipes_loaded, + "index_count": index_count, + "recipe_count": recipe_count, + "artifact_mismatch": artifact_mismatch, + "available": index_loaded and recipes_loaded, + } + + def _matches_filters(self, recipe: Dict[str, Any], filters: Optional[RecipeFilters]) -> bool: + if not filters: + return True + + if filters.cuisine and not _list_overlaps(recipe.get("cuisine"), filters.cuisine): + return False + if filters.meal_type and not _list_overlaps(recipe.get("meal_type"), filters.meal_type): + return False + if filters.dietary_tags and not _list_overlaps(recipe.get("dietary_tags"), filters.dietary_tags): + return False + + if not _in_range(_get_numeric(recipe, "calories"), filters.min_calories, filters.max_calories): + return False + if not _in_range(_get_numeric(recipe, "protein_g"), filters.min_protein_g, filters.max_protein_g): + return False + if not _in_range(_get_numeric(recipe, "carbs_g"), filters.min_carbs_g, filters.max_carbs_g): + return False + if not _in_range(_get_numeric(recipe, "fat_g"), filters.min_fat_g, filters.max_fat_g): + return False + + if filters.max_prep_time_minutes is not None: + prep_time = _get_int(recipe, "prep_time_minutes") + if prep_time is None or prep_time > filters.max_prep_time_minutes: + return False + + if filters.max_total_time_minutes is not None: + total_time = _get_int(recipe, "total_time_minutes") + if total_time is None or total_time > filters.max_total_time_minutes: + return False + + return True + + def _rerank_candidates( + self, + candidates: List[Tuple[Dict[str, Any], float]], + query_ingredients: List[str], + ) -> List[Tuple[Dict[str, Any], float]]: + reranked: List[Tuple[Dict[str, Any], float]] = [] + query_set = set(query_ingredients) + for recipe, score in candidates: + recipe_ingredients = normalize_ingredient_list(recipe.get("ingredients") or []) + overlap = _overlap_score(query_set, set(recipe_ingredients)) + combined_score = (0.85 * score) + (0.15 * overlap) + reranked.append((recipe, combined_score)) + + reranked.sort(key=lambda item: item[1], reverse=True) + return reranked + + @staticmethod + def _recipe_to_result(recipe: Dict[str, Any], score: float) -> Dict[str, Any]: + return { + "id": str(recipe.get("id", "")), + "title": recipe.get("title", ""), + "ingredients": recipe.get("ingredients") or [], + "instructions": recipe.get("instructions") or [], + "calories": recipe.get("calories"), + "protein_g": recipe.get("protein_g"), + "carbs_g": recipe.get("carbs_g"), + "fat_g": recipe.get("fat_g"), + "cuisine": recipe.get("cuisine"), + "meal_type": recipe.get("meal_type"), + "dietary_tags": recipe.get("dietary_tags"), + "allergens": recipe.get("allergens"), + "prep_time_minutes": recipe.get("prep_time_minutes"), + "cook_time_minutes": recipe.get("cook_time_minutes"), + "total_time_minutes": recipe.get("total_time_minutes"), + "score": float(score), + } + + @staticmethod + def _build_diagnostics( + query: str, + normalized_query: str, + candidate_k: int, + returned_k: int, + filters: Optional[RecipeFilters], + reranked: bool, + top_ids: List[str], + top_scores: List[float], + ) -> RetrievalDiagnostics: + return RetrievalDiagnostics( + query=query, + normalized_query=normalized_query, + candidate_k=candidate_k, + returned_k=returned_k, + reranked=reranked, + filters=filters, + top_ids=top_ids, + top_scores=top_scores, + ) + + +def _list_overlaps(left: Any, right: Any) -> bool: + left_list = _normalize_list(left) + right_list = _normalize_list(right) + return bool(set(left_list).intersection(right_list)) + + +def _normalize_list(value: Any) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v).lower() for v in value if str(v).strip()] + if isinstance(value, str): + if "," in value: + return [v.strip().lower() for v in value.split(",") if v.strip()] + return [value.strip().lower()] + return [str(value).lower()] + + +def _in_range(value: Optional[float], min_value: Optional[float], max_value: Optional[float]) -> bool: + if value is None: + return min_value is None and max_value is None + if min_value is not None and value < min_value: + return False + if max_value is not None and value > max_value: + return False + return True + + +def _get_numeric(recipe: Dict[str, Any], key: str) -> Optional[float]: + if key in recipe and recipe.get(key) is not None: + try: + return float(recipe[key]) + except (TypeError, ValueError): + return None + macros = recipe.get("macros") or {} + if isinstance(macros, dict) and key in macros and macros.get(key) is not None: + try: + return float(macros[key]) + except (TypeError, ValueError): + return None + return None + + +def _get_int(recipe: Dict[str, Any], key: str) -> Optional[int]: + if key in recipe and recipe.get(key) is not None: + try: + return int(recipe[key]) + except (TypeError, ValueError): + return None + times = recipe.get("times") or {} + if isinstance(times, dict) and key in times and times.get(key) is not None: + try: + return int(times[key]) + except (TypeError, ValueError): + return None + return None + + +def _overlap_score(query_set: set[str], recipe_set: set[str]) -> float: + if not query_set: + return 0.0 + return len(query_set.intersection(recipe_set)) / float(len(query_set)) diff --git a/backend/app/services/recipe_service.py b/backend/app/services/recipe_service.py index 833cd45..4c966ec 100644 --- a/backend/app/services/recipe_service.py +++ b/backend/app/services/recipe_service.py @@ -1,5 +1,6 @@ -from typing import List -from ..schemas.recipe import Recipe +from typing import List, Optional, Tuple + +from ..schemas.recipe import Recipe, RecipeFilters, RetrievalDiagnostics from .rag_service import RagService @@ -7,6 +8,66 @@ class RecipeService: def __init__(self, rag: RagService): self.rag = rag - def search_recipes(self, query: str, top_k: int) -> List[Recipe]: - raw = self.rag.search(query=query, top_k=top_k) - return [Recipe(**r) for r in raw] \ No newline at end of file + def search_recipes( + self, + query: str, + top_k: int, + filters: Optional[RecipeFilters] = None, + rerank: bool = True, + include_diagnostics: bool = False, + candidate_k: Optional[int] = None, + ) -> Tuple[List[Recipe], Optional[RetrievalDiagnostics]]: + if include_diagnostics: + raw, diagnostics = self.rag.search_with_diagnostics( + query=query, + top_k=top_k, + filters=filters, + rerank=rerank, + candidate_k=candidate_k, + include_diagnostics=True, + ) + else: + raw = self.rag.search( + query=query, + top_k=top_k, + filters=filters, + rerank=rerank, + candidate_k=candidate_k, + ) + diagnostics = None + + return [Recipe(**r) for r in raw], diagnostics + + def recommend_recipes( + self, + ingredients: List[str], + dietary_notes: Optional[str], + top_k: int, + filters: Optional[RecipeFilters] = None, + rerank: bool = True, + include_diagnostics: bool = False, + candidate_k: Optional[int] = None, + ) -> Tuple[List[Recipe], Optional[RetrievalDiagnostics]]: + query = f"{', '.join(ingredients)}. {dietary_notes or ''}".strip() + if include_diagnostics: + raw, diagnostics = self.rag.search_with_diagnostics( + query=query, + top_k=top_k, + filters=filters, + rerank=rerank, + candidate_k=candidate_k, + ingredients=ingredients, + include_diagnostics=True, + ) + else: + raw = self.rag.search( + query=query, + top_k=top_k, + filters=filters, + rerank=rerank, + candidate_k=candidate_k, + ingredients=ingredients, + ) + diagnostics = None + + return [Recipe(**r) for r in raw], diagnostics diff --git a/backend/app/services/retrieval_service.py b/backend/app/services/retrieval_service.py new file mode 100644 index 0000000..21a37d8 --- /dev/null +++ b/backend/app/services/retrieval_service.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import logging +from typing import Dict, Iterable, List, Optional, Tuple + +from ..core.config import settings +from ..repositories.lexical_index import LexicalIndex, LexicalSearchResult +from ..repositories.vector_store import QdrantVectorStore +from ..schemas.retrieval import ( + RetrievalDiagnostics, + RetrievalMode, + RetrievalRequest, + RetrievalResponse, + RetrievedChunk, +) +from .embedding_service import EmbeddingService +from .filters import build_qdrant_filter + +logger = logging.getLogger("nourishai.retrieval") + + +class RetrievalService: + def __init__( + self, + vector_store: QdrantVectorStore, + lexical_index: LexicalIndex, + embedder: EmbeddingService, + hybrid_alpha: float, + ): + self.vector_store = vector_store + self.lexical_index = lexical_index + self.embedder = embedder + self.hybrid_alpha = hybrid_alpha + + def search(self, request: RetrievalRequest) -> RetrievalResponse: + query = request.query.strip() + if not query: + return RetrievalResponse(query=request.query, mode=request.mode, results=[]) + + doc_types = [dt.value if hasattr(dt, "value") else str(dt) for dt in (request.doc_types or [])] + candidate_k = request.candidate_k or max(request.top_k, settings.RETRIEVAL_CANDIDATES) + + dense_results = [] + keyword_results: List[LexicalSearchResult] = [] + + qdrant_filter = build_qdrant_filter(request.filters, doc_types or None) + + if request.mode in (RetrievalMode.dense, RetrievalMode.hybrid): + vector = self.embedder.embed([query])[0] + dense_results = self.vector_store.search(vector, limit=candidate_k, qdrant_filter=qdrant_filter) + + if request.mode in (RetrievalMode.keyword, RetrievalMode.hybrid): + keyword_results = self.lexical_index.search( + query=query, + top_k=candidate_k, + filters=request.filters, + doc_types=doc_types or None, + ) + + results = self._merge_results( + request=request, + dense_results=dense_results, + keyword_results=keyword_results, + ) + + diagnostics = None + if request.include_diagnostics: + diagnostics = RetrievalDiagnostics( + mode=request.mode, + dense_candidates=len(dense_results) if dense_results else 0, + keyword_candidates=len(keyword_results) if keyword_results else 0, + candidate_k=candidate_k, + collection=settings.QDRANT_COLLECTION, + lexical_index_loaded=self.lexical_index.available, + hybrid_alpha=self.hybrid_alpha, + filters=request.filters.model_dump() if request.filters else None, + ) + + return RetrievalResponse( + query=request.query, + mode=request.mode, + results=results[: request.top_k], + diagnostics=diagnostics, + ) + + def _merge_results( + self, + request: RetrievalRequest, + dense_results: Iterable, + keyword_results: List[LexicalSearchResult], + ) -> List[RetrievedChunk]: + dense_map: Dict[str, Tuple[float, dict]] = {} + for point in dense_results: + payload = point.payload or {} + dense_map[str(point.id)] = (float(point.score), payload) + + keyword_map: Dict[str, Tuple[float, dict, str]] = {} + for match in keyword_results: + keyword_map[match.chunk_id] = (match.score, match.payload, match.text) + + if request.mode == RetrievalMode.dense: + return [ + self._payload_to_chunk( + str(point.id), + float(point.score), + point.payload or {}, + dense_score=float(point.score), + ) + for point in dense_results + ] + + if request.mode == RetrievalMode.keyword: + return [ + self._payload_to_chunk( + match.chunk_id, + match.score, + match.payload, + dense_score=None, + keyword_score=match.score, + text_override=match.text, + ) + for match in keyword_results + ] + + dense_rank = _rank_scores(dense_map) + keyword_rank = _rank_scores({cid: (score, payload) for cid, (score, payload, _) in keyword_map.items()}) + + combined: List[RetrievedChunk] = [] + for chunk_id in set(dense_rank.keys()).union(keyword_rank.keys()): + dense_score, dense_payload = dense_map.get(chunk_id, (None, {})) + keyword_entry = keyword_map.get(chunk_id) + keyword_score = keyword_entry[0] if keyword_entry else None + keyword_payload = keyword_entry[1] if keyword_entry else {} + text_override = keyword_entry[2] if keyword_entry else None + + payload = dense_payload or keyword_payload + + if request.rerank: + combined_score = self.hybrid_alpha * dense_rank.get(chunk_id, 0.0) + (1 - self.hybrid_alpha) * keyword_rank.get(chunk_id, 0.0) + else: + combined_score = dense_score if dense_score is not None else (keyword_score or 0.0) + + combined.append( + self._payload_to_chunk( + chunk_id, + combined_score, + payload, + dense_score=dense_score, + keyword_score=keyword_score, + text_override=text_override, + ) + ) + + combined.sort(key=lambda item: item.score, reverse=True) + return combined + + @staticmethod + def _payload_to_chunk( + chunk_id: str, + score: float, + payload: dict, + dense_score: Optional[float] = None, + keyword_score: Optional[float] = None, + text_override: Optional[str] = None, + ) -> RetrievedChunk: + text = text_override or payload.get("text") or "" + metadata = dict(payload) + metadata.pop("text", None) + return RetrievedChunk( + chunk_id=chunk_id, + doc_id=str(payload.get("doc_id") or ""), + doc_type=str(payload.get("doc_type") or ""), + title=payload.get("title"), + text=text, + source=payload.get("source"), + score=float(score), + dense_score=float(dense_score) if dense_score is not None else None, + keyword_score=float(keyword_score) if keyword_score is not None else None, + metadata=metadata, + ) + + +def _rank_scores(items: Dict[str, Tuple[float, dict]]) -> Dict[str, float]: + sorted_items = sorted(items.items(), key=lambda item: item[1][0], reverse=True) + ranks: Dict[str, float] = {} + for idx, (chunk_id, _) in enumerate(sorted_items): + ranks[chunk_id] = 1.0 / (idx + 1) + return ranks diff --git a/backend/requirements.txt b/backend/requirements.txt index 34ef2eb..fc0cf82 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,12 +1,16 @@ -fastapi==0.111.0 -uvicorn[standard]==0.29.0 -pydantic==2.7.1 -pydantic-settings==2.2.1 +fastapi==0.135.1 +uvicorn[standard]==0.41.0 +pydantic==2.12.5 +pydantic-settings==2.13.1 python-multipart==0.0.9 -requests==2.32.3 -numpy==1.26.4 -scikit-learn==1.4.2 -Pillow==10.3.0 -gdown==5.1.0 +requests==2.32.5 +numpy==2.2.6 +Pillow==12.1.1 +gdown==5.2.1 sentence-transformers==2.7.0 -# faiss-cpu==1.8.0 # Uncomment when FAISS index is available +faiss-cpu==1.13.2 +qdrant-client==1.8.2 +rank-bm25==0.2.2 +openai==2.28.0 +pytest==9.0.2 +httpx==0.28.1 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..23c6690 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,6 @@ +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) diff --git a/backend/tests/test_endpoints.py b/backend/tests/test_endpoints.py new file mode 100644 index 0000000..3bbb2bf --- /dev/null +++ b/backend/tests/test_endpoints.py @@ -0,0 +1,67 @@ +from fastapi.testclient import TestClient + +from backend.app.api import deps +from backend.app.main import app +from backend.app.schemas.ai import ( + GroundedRecommendationResponse, + MealPlanResponse, + NutritionQAResponse, + RecommendedRecipe, + SourceRecipeRef, +) + + +class FakeGroundedService: + def grounded_recommendations(self, payload): + return GroundedRecommendationResponse( + recommended_recipes=[ + RecommendedRecipe(recipe_id="rec-1", title="Test", why_selected="fits", substitutions=[], warnings=[]) + ], + warnings=[], + source_recipes=[SourceRecipeRef(id="rec-1", title="Test")], + ) + + def generate_meal_plan(self, payload): + return MealPlanResponse(days=[], warnings=[], source_recipes=[]) + + def answer_nutrition_question(self, payload): + return NutritionQAResponse(answer="Test", supporting_sources=[], warnings=[]) + + +def test_grounded_recommend_endpoint(): + app.dependency_overrides[deps.get_grounded_ai_service] = lambda: FakeGroundedService() + client = TestClient(app) + response = client.post( + "/api/v1/recipes/recommend/grounded", + json={"ingredients": ["chicken"], "top_k": 1}, + ) + assert response.status_code == 200 + data = response.json() + assert "recommended_recipes" in data + app.dependency_overrides = {} + + +def test_meal_plan_endpoint(): + app.dependency_overrides[deps.get_grounded_ai_service] = lambda: FakeGroundedService() + client = TestClient(app) + response = client.post( + "/api/v1/meal-plans/generate", + json={"days": 2, "meals_per_day": 2}, + ) + assert response.status_code == 200 + data = response.json() + assert "days" in data + app.dependency_overrides = {} + + +def test_nutrition_qa_endpoint(): + app.dependency_overrides[deps.get_grounded_ai_service] = lambda: FakeGroundedService() + client = TestClient(app) + response = client.post( + "/api/v1/nutrition/qa", + json={"question": "How much protein?"}, + ) + assert response.status_code == 200 + data = response.json() + assert "answer" in data + app.dependency_overrides = {} diff --git a/backend/tests/test_grounded_fallback.py b/backend/tests/test_grounded_fallback.py new file mode 100644 index 0000000..126cfbf --- /dev/null +++ b/backend/tests/test_grounded_fallback.py @@ -0,0 +1,20 @@ +from backend.app.schemas.ai import GroundedRecommendationRequest +from backend.app.services.grounded_ai_service import GroundedAIService + + +class FakeRagService: + def search_with_diagnostics(self, **kwargs): + return [], None + + +class FakeLLMService: + def generate_structured(self, *args, **kwargs): + raise AssertionError("LLM should not be called when retrieval is empty") + + +def test_grounded_recommendation_fallback(): + service = GroundedAIService(rag=FakeRagService(), llm=FakeLLMService()) + req = GroundedRecommendationRequest(ingredients=["chicken"], top_k=3) + resp = service.grounded_recommendations(req) + assert resp.recommended_recipes == [] + assert resp.warnings diff --git a/backend/tests/test_grounded_llm_failure.py b/backend/tests/test_grounded_llm_failure.py new file mode 100644 index 0000000..7513ee7 --- /dev/null +++ b/backend/tests/test_grounded_llm_failure.py @@ -0,0 +1,28 @@ +from backend.app.schemas.ai import GroundedRecommendationRequest +from backend.app.services.grounded_ai_service import GroundedAIService + + +class FakeRag: + def search_with_diagnostics(self, **kwargs): + return [ + { + "id": "rec-1", + "title": "Test Recipe", + "ingredients": ["chicken"], + "instructions": ["Cook"], + "calories": 300, + } + ], None + + +class FailingLLM: + def generate_structured(self, *args, **kwargs): + raise RuntimeError("LLM failure") + + +def test_grounded_recommendation_llm_failure_returns_fallback(): + service = GroundedAIService(rag=FakeRag(), llm=FailingLLM()) + request = GroundedRecommendationRequest(ingredients=["chicken"], top_k=1) + response = service.grounded_recommendations(request) + assert response.recommended_recipes == [] + assert response.warnings diff --git a/backend/tests/test_health_endpoints.py b/backend/tests/test_health_endpoints.py new file mode 100644 index 0000000..a3db33e --- /dev/null +++ b/backend/tests/test_health_endpoints.py @@ -0,0 +1,76 @@ +from fastapi.testclient import TestClient + +from backend.app.api import deps +from backend.app.main import app + + +class FakeRagReady: + def status(self): + return { + "index_loaded": True, + "recipes_loaded": True, + "index_count": 10, + "recipe_count": 10, + "artifact_mismatch": False, + "available": True, + } + + +class FakeRagNotReady: + def status(self): + return { + "index_loaded": False, + "recipes_loaded": False, + "index_count": 0, + "recipe_count": 0, + "artifact_mismatch": False, + "available": False, + } + + +class FakeLLM: + model = "gpt-test" + + def is_configured(self): + return True + + +def test_health_endpoint_includes_request_id(): + client = TestClient(app) + response = client.get("/api/v1/health") + assert response.status_code == 200 + assert response.headers.get("X-Request-ID") + assert response.json()["status"] == "ok" + + +def test_ready_endpoint_reports_ready(): + app.dependency_overrides[deps.get_rag_service] = lambda: FakeRagReady() + app.dependency_overrides[deps.get_llm_service] = lambda: FakeLLM() + client = TestClient(app) + response = client.get("/api/v1/ready") + assert response.status_code == 200 + payload = response.json() + assert payload["ready"] is True + app.dependency_overrides = {} + + +def test_ready_endpoint_reports_not_ready(): + app.dependency_overrides[deps.get_rag_service] = lambda: FakeRagNotReady() + app.dependency_overrides[deps.get_llm_service] = lambda: FakeLLM() + client = TestClient(app) + response = client.get("/api/v1/ready") + assert response.status_code == 503 + payload = response.json() + assert payload["ready"] is False + app.dependency_overrides = {} + + +def test_status_endpoint_alias(): + app.dependency_overrides[deps.get_rag_service] = lambda: FakeRagReady() + app.dependency_overrides[deps.get_llm_service] = lambda: FakeLLM() + client = TestClient(app) + response = client.get("/api/v1/status") + assert response.status_code == 200 + payload = response.json() + assert "ready" in payload + app.dependency_overrides = {} diff --git a/backend/tests/test_ingredient_normalizer.py b/backend/tests/test_ingredient_normalizer.py new file mode 100644 index 0000000..ce3a9fc --- /dev/null +++ b/backend/tests/test_ingredient_normalizer.py @@ -0,0 +1,15 @@ +from backend.app.services.ingredient_normalizer import normalize_ingredient, normalize_ingredient_list + + +def test_normalize_ingredient_aliases(): + assert normalize_ingredient("Garbanzo Beans") == "chickpea" + assert normalize_ingredient("Scallions") == "green onion" + + +def test_normalize_ingredient_plural(): + assert normalize_ingredient("tomatoes") == "tomato" + + +def test_normalize_list(): + values = normalize_ingredient_list(["Chicken breasts", "BELL peppers"]) + assert values == ["chicken breast", "bell pepper"] diff --git a/backend/tests/test_invalid_filters.py b/backend/tests/test_invalid_filters.py new file mode 100644 index 0000000..38f013c --- /dev/null +++ b/backend/tests/test_invalid_filters.py @@ -0,0 +1,16 @@ +from fastapi.testclient import TestClient + +from backend.app.main import app + + +def test_search_invalid_filters_returns_422(): + client = TestClient(app) + response = client.post( + "/api/v1/recipes/search", + json={ + "query": "chicken", + "top_k": 3, + "filters": {"min_calories": "not-a-number"}, + }, + ) + assert response.status_code == 422 diff --git a/backend/tests/test_llm_service.py b/backend/tests/test_llm_service.py new file mode 100644 index 0000000..59d3a66 --- /dev/null +++ b/backend/tests/test_llm_service.py @@ -0,0 +1,50 @@ +import json + +from backend.app.schemas.ai import LLMRecommendationPayload, RecommendedRecipe +from backend.app.services.llm_service import LLMService + + +class DummyMessage: + def __init__(self, content: str): + self.content = content + + +class DummyChoice: + def __init__(self, content: str): + self.message = DummyMessage(content) + + +class DummyResponse: + def __init__(self, content: str): + self.choices = [DummyChoice(content)] + + +class DummyCompletions: + def __init__(self, content: str): + self._content = content + + def create(self, **kwargs): + return DummyResponse(self._content) + + +class DummyChat: + def __init__(self, content: str): + self.completions = DummyCompletions(content) + + +class DummyClient: + def __init__(self, content: str): + self.chat = DummyChat(content) + + +def test_llm_service_structured_output(): + payload = LLMRecommendationPayload( + recommended_recipes=[RecommendedRecipe(recipe_id="rec-1", title="Test", why_selected="fits", substitutions=[], warnings=[])], + warnings=[], + ) + content = json.dumps(payload.model_dump()) + client = DummyClient(content) + + service = LLMService(api_key="test", client=client) + result = service.generate_structured("sys", "user", LLMRecommendationPayload) + assert result.recommended_recipes[0].recipe_id == "rec-1" diff --git a/backend/tests/test_rag_filters.py b/backend/tests/test_rag_filters.py new file mode 100644 index 0000000..ad8d089 --- /dev/null +++ b/backend/tests/test_rag_filters.py @@ -0,0 +1,24 @@ +from backend.app.schemas.recipe import RecipeFilters +from backend.app.services.rag_service import RagService + + +def test_metadata_filtering(): + service = RagService("missing.index", "missing.pkl") + recipe = { + "id": "rec-1", + "title": "Test", + "cuisine": ["italian"], + "meal_type": ["dinner"], + "dietary_tags": ["vegetarian"], + "calories": 450, + "protein_g": 20, + } + + filters = RecipeFilters(cuisine=["italian"], min_calories=300, max_calories=500) + assert service._matches_filters(recipe, filters) is True + + filters = RecipeFilters(cuisine=["mexican"]) + assert service._matches_filters(recipe, filters) is False + + filters = RecipeFilters(max_calories=300) + assert service._matches_filters(recipe, filters) is False diff --git a/backend/tests/test_rag_missing_index.py b/backend/tests/test_rag_missing_index.py new file mode 100644 index 0000000..4db04ad --- /dev/null +++ b/backend/tests/test_rag_missing_index.py @@ -0,0 +1,10 @@ +import pytest +from starlette.exceptions import HTTPException + +from backend.app.services.rag_service import RagService + + +def test_rag_missing_index_raises(): + service = RagService("missing.index", "missing.pkl") + with pytest.raises(HTTPException): + service.search(query="test", top_k=3) diff --git a/backend/tests/test_startup_validation.py b/backend/tests/test_startup_validation.py new file mode 100644 index 0000000..791b6bf --- /dev/null +++ b/backend/tests/test_startup_validation.py @@ -0,0 +1,50 @@ +from backend.app.core.validation import validate_startup + + +class FakeRag: + def __init__(self, status): + self._status = status + + def status(self): + return self._status + + +class FakeLLM: + def __init__(self, configured: bool): + self._configured = configured + + def is_configured(self): + return self._configured + + +def test_validate_startup_reports_missing_artifacts(): + rag = FakeRag( + { + "index_loaded": False, + "recipes_loaded": False, + "index_count": 0, + "recipe_count": 0, + "artifact_mismatch": False, + "available": False, + } + ) + llm = FakeLLM(configured=False) + report = validate_startup(rag, llm) + assert report["errors"] + assert report["warnings"] + + +def test_validate_startup_warns_on_mismatch(): + rag = FakeRag( + { + "index_loaded": True, + "recipes_loaded": True, + "index_count": 10, + "recipe_count": 8, + "artifact_mismatch": True, + "available": True, + } + ) + llm = FakeLLM(configured=True) + report = validate_startup(rag, llm) + assert report["warnings"] diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..56a07dd --- /dev/null +++ b/data/README.md @@ -0,0 +1,34 @@ +# NourishAI Data + +This folder holds raw documents for ingestion. Each subfolder can contain one or more JSON files. + +## recipes/*.json +Expected fields per recipe (flexible): +- id or recipe_id +- title or name +- ingredients (list or comma-separated string) +- instructions or steps (list or string) +- meal_type (list) +- dietary_tags (list) +- cuisine (list) +- nutrition (object with calories, protein_g, carbs_g, fat_g) +- source (string) + +## nutrition/*.json +Expected fields per reference: +- id +- title +- text +- nutrients (list) +- source + +## ingredients/*.json +Expected fields per ingredient: +- id +- name +- aliases (list) +- notes +- substitutions (list) +- allergens (list) +- nutrition (object with calories, protein_g, carbs_g, fat_g) +- source diff --git a/data/ingredients/sample_ingredients.json b/data/ingredients/sample_ingredients.json new file mode 100644 index 0000000..57e8347 --- /dev/null +++ b/data/ingredients/sample_ingredients.json @@ -0,0 +1,47 @@ +[ + { + "id": "ing_001", + "name": "quinoa", + "aliases": ["quinua"], + "notes": "Gluten-free grain-like seed with a mild nutty flavor.", + "substitutions": ["brown rice", "farro"], + "allergens": [], + "nutrition": { + "calories": 222, + "protein_g": 8, + "carbs_g": 39, + "fat_g": 4 + }, + "source": "sample" + }, + { + "id": "ing_002", + "name": "chickpeas", + "aliases": ["garbanzo beans"], + "notes": "Legume used in curries, salads, and hummus.", + "substitutions": ["white beans", "lentils"], + "allergens": [], + "nutrition": { + "calories": 269, + "protein_g": 15, + "carbs_g": 45, + "fat_g": 4 + }, + "source": "sample" + }, + { + "id": "ing_003", + "name": "chicken breast", + "aliases": ["boneless chicken"], + "notes": "Lean protein commonly used in bowls and salads.", + "substitutions": ["turkey breast", "tofu"], + "allergens": [], + "nutrition": { + "calories": 165, + "protein_g": 31, + "carbs_g": 0, + "fat_g": 3 + }, + "source": "sample" + } +] diff --git a/data/nutrition/sample_nutrition.json b/data/nutrition/sample_nutrition.json new file mode 100644 index 0000000..f8a84bb --- /dev/null +++ b/data/nutrition/sample_nutrition.json @@ -0,0 +1,23 @@ +[ + { + "id": "nutri_001", + "title": "Protein Intake Overview", + "text": "Protein supports muscle maintenance and satiety. Distribute protein across meals and include a variety of sources such as lean poultry, legumes, dairy, and tofu.", + "nutrients": ["protein"], + "source": "sample" + }, + { + "id": "nutri_002", + "title": "Fiber and Blood Sugar", + "text": "Dietary fiber slows digestion and can help moderate post-meal blood sugar spikes. Pair fiber-rich foods like legumes, whole grains, and vegetables with balanced protein and fat.", + "nutrients": ["fiber", "carbohydrates"], + "source": "sample" + }, + { + "id": "nutri_003", + "title": "Sodium Awareness", + "text": "Reducing sodium intake can support heart health. Favor herbs, citrus, and spice blends over salt-heavy seasonings when possible.", + "nutrients": ["sodium"], + "source": "sample" + } +] diff --git a/data/recipes/sample_recipes.json b/data/recipes/sample_recipes.json new file mode 100644 index 0000000..e522a0b --- /dev/null +++ b/data/recipes/sample_recipes.json @@ -0,0 +1,93 @@ +[ + { + "id": "rec_001", + "title": "Lemon Herb Chicken Bowl", + "meal_type": ["dinner"], + "dietary_tags": ["high_protein", "gluten_free"], + "cuisine": ["mediterranean"], + "ingredients": [ + "chicken breast", + "lemon", + "olive oil", + "garlic", + "oregano", + "quinoa", + "spinach", + "cherry tomatoes" + ], + "instructions": [ + "Marinate chicken with lemon, olive oil, garlic, and oregano for 20 minutes.", + "Cook quinoa until fluffy; set aside.", + "Sear chicken until cooked through, then slice.", + "Assemble bowl with quinoa, spinach, tomatoes, and sliced chicken.", + "Drizzle with remaining marinade as dressing." + ], + "nutrition": { + "calories": 520, + "protein_g": 45, + "carbs_g": 42, + "fat_g": 18 + }, + "source": "sample" + }, + { + "id": "rec_002", + "title": "Chickpea Spinach Curry", + "meal_type": ["dinner"], + "dietary_tags": ["vegan", "high_fiber"], + "cuisine": ["indian"], + "ingredients": [ + "chickpeas", + "spinach", + "onion", + "garlic", + "ginger", + "tomato puree", + "cumin", + "coriander", + "turmeric" + ], + "instructions": [ + "Saute onion, garlic, and ginger until fragrant.", + "Add spices and toast for 30 seconds.", + "Stir in tomato puree and simmer for 5 minutes.", + "Add chickpeas and cook for 10 minutes.", + "Fold in spinach until wilted and season to taste." + ], + "nutrition": { + "calories": 410, + "protein_g": 18, + "carbs_g": 52, + "fat_g": 10 + }, + "source": "sample" + }, + { + "id": "rec_003", + "title": "Overnight Oats with Berries", + "meal_type": ["breakfast"], + "dietary_tags": ["vegetarian", "high_fiber"], + "cuisine": ["american"], + "ingredients": [ + "rolled oats", + "milk", + "greek yogurt", + "chia seeds", + "blueberries", + "strawberries", + "honey" + ], + "instructions": [ + "Combine oats, milk, yogurt, and chia seeds in a jar.", + "Refrigerate overnight.", + "Top with berries and honey before serving." + ], + "nutrition": { + "calories": 360, + "protein_g": 17, + "carbs_g": 48, + "fat_g": 9 + }, + "source": "sample" + } +] diff --git a/data/sample_recipes.json b/data/sample_recipes.json new file mode 100644 index 0000000..a4d455b --- /dev/null +++ b/data/sample_recipes.json @@ -0,0 +1,172 @@ +[ + { + "id": "rec-001", + "title": "Pasta Primavera", + "ingredients": ["pasta", "zucchini", "cherry tomatoes", "garlic", "olive oil", "parmesan"], + "instructions": ["Boil pasta until al dente.", "Saute zucchini and tomatoes with garlic.", "Toss pasta with vegetables and parmesan."], + "calories": 450.0, + "protein_g": 16.0, + "carbs_g": 62.0, + "fat_g": 14.0, + "cuisine": ["italian"], + "meal_type": ["dinner"], + "dietary_tags": ["vegetarian"], + "allergens": ["dairy", "gluten"], + "prep_time_minutes": 15, + "cook_time_minutes": 20, + "total_time_minutes": 35 + }, + { + "id": "rec-002", + "title": "Lemon Garlic Chicken", + "ingredients": ["chicken breast", "lemon", "garlic", "olive oil", "oregano"], + "instructions": ["Marinate chicken with lemon, garlic, and oregano.", "Grill until cooked through."], + "calories": 390.0, + "protein_g": 42.0, + "carbs_g": 6.0, + "fat_g": 18.0, + "cuisine": ["mediterranean"], + "meal_type": ["dinner"], + "dietary_tags": ["high_protein", "gluten_free", "low_carb"], + "allergens": [], + "prep_time_minutes": 10, + "cook_time_minutes": 15, + "total_time_minutes": 25 + }, + { + "id": "rec-003", + "title": "Chickpea Spinach Stew", + "ingredients": ["chickpeas", "spinach", "onion", "tomato paste", "cumin"], + "instructions": ["Saute onion with cumin.", "Add tomato paste and chickpeas.", "Simmer and stir in spinach."], + "calories": 410.0, + "protein_g": 18.0, + "carbs_g": 54.0, + "fat_g": 10.0, + "cuisine": ["middle_eastern"], + "meal_type": ["dinner"], + "dietary_tags": ["vegan", "high_fiber"], + "allergens": [], + "prep_time_minutes": 12, + "cook_time_minutes": 25, + "total_time_minutes": 37 + }, + { + "id": "rec-004", + "title": "Avocado Egg Toast", + "ingredients": ["whole grain bread", "avocado", "egg", "lemon juice", "salt", "pepper"], + "instructions": ["Toast bread.", "Mash avocado with lemon juice.", "Top with a fried or poached egg."], + "calories": 320.0, + "protein_g": 14.0, + "carbs_g": 28.0, + "fat_g": 18.0, + "cuisine": ["american"], + "meal_type": ["breakfast"], + "dietary_tags": ["vegetarian"], + "allergens": ["gluten", "egg"], + "prep_time_minutes": 5, + "cook_time_minutes": 6, + "total_time_minutes": 11 + }, + { + "id": "rec-005", + "title": "Quinoa Buddha Bowl", + "ingredients": ["quinoa", "roasted sweet potato", "kale", "chickpeas", "tahini"], + "instructions": ["Cook quinoa.", "Roast sweet potato.", "Assemble bowl with kale, chickpeas, and tahini drizzle."], + "calories": 500.0, + "protein_g": 20.0, + "carbs_g": 68.0, + "fat_g": 18.0, + "cuisine": ["mediterranean"], + "meal_type": ["lunch"], + "dietary_tags": ["vegan", "high_fiber"], + "allergens": ["sesame"], + "prep_time_minutes": 15, + "cook_time_minutes": 25, + "total_time_minutes": 40 + }, + { + "id": "rec-006", + "title": "Turkey Lettuce Wraps", + "ingredients": ["ground turkey", "lettuce", "soy sauce", "ginger", "garlic", "carrots"], + "instructions": ["Cook turkey with ginger and garlic.", "Stir in carrots and soy sauce.", "Serve in lettuce cups."], + "calories": 360.0, + "protein_g": 30.0, + "carbs_g": 16.0, + "fat_g": 18.0, + "cuisine": ["asian"], + "meal_type": ["dinner"], + "dietary_tags": ["high_protein", "low_carb"], + "allergens": ["soy"], + "prep_time_minutes": 10, + "cook_time_minutes": 15, + "total_time_minutes": 25 + }, + { + "id": "rec-007", + "title": "Greek Yogurt Parfait", + "ingredients": ["greek yogurt", "granola", "blueberries", "honey"], + "instructions": ["Layer yogurt, granola, and berries.", "Drizzle with honey."], + "calories": 280.0, + "protein_g": 17.0, + "carbs_g": 34.0, + "fat_g": 8.0, + "cuisine": ["greek"], + "meal_type": ["breakfast", "snack"], + "dietary_tags": ["vegetarian"], + "allergens": ["dairy", "gluten"], + "prep_time_minutes": 5, + "cook_time_minutes": 0, + "total_time_minutes": 5 + }, + { + "id": "rec-008", + "title": "Salmon with Herb Rice", + "ingredients": ["salmon", "brown rice", "parsley", "lemon", "olive oil"], + "instructions": ["Bake salmon with lemon.", "Cook rice and stir in parsley.", "Serve together."], + "calories": 520.0, + "protein_g": 38.0, + "carbs_g": 46.0, + "fat_g": 20.0, + "cuisine": ["american"], + "meal_type": ["dinner"], + "dietary_tags": ["high_protein", "gluten_free"], + "allergens": ["fish"], + "prep_time_minutes": 10, + "cook_time_minutes": 25, + "total_time_minutes": 35 + }, + { + "id": "rec-009", + "title": "Vegetable Stir Fry", + "ingredients": ["broccoli", "bell pepper", "snap peas", "soy sauce", "sesame oil"], + "instructions": ["Stir fry vegetables.", "Add soy sauce and sesame oil.", "Serve hot."], + "calories": 300.0, + "protein_g": 10.0, + "carbs_g": 32.0, + "fat_g": 14.0, + "cuisine": ["asian"], + "meal_type": ["dinner"], + "dietary_tags": ["vegan"], + "allergens": ["soy", "sesame"], + "prep_time_minutes": 10, + "cook_time_minutes": 10, + "total_time_minutes": 20 + }, + { + "id": "rec-010", + "title": "Berry Overnight Oats", + "ingredients": ["rolled oats", "milk", "chia seeds", "strawberries", "blueberries"], + "instructions": ["Combine oats, milk, and chia seeds.", "Refrigerate overnight.", "Top with berries before serving."], + "calories": 340.0, + "protein_g": 14.0, + "carbs_g": 52.0, + "fat_g": 8.0, + "cuisine": ["american"], + "meal_type": ["breakfast"], + "dietary_tags": ["vegetarian", "high_fiber"], + "allergens": ["dairy", "gluten"], + "prep_time_minutes": 5, + "cook_time_minutes": 0, + "total_time_minutes": 5 + } +] diff --git a/docker-compose.yml b/docker-compose.yml index cd79bb6..74f546f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,4 +11,4 @@ services: ports: - "8501:8501" depends_on: - - api \ No newline at end of file + - api diff --git a/eval/__init__.py b/eval/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/eval/__init__.py @@ -0,0 +1 @@ + diff --git a/eval/datasets/grounded_generation_eval.json b/eval/datasets/grounded_generation_eval.json new file mode 100644 index 0000000..9894df1 --- /dev/null +++ b/eval/datasets/grounded_generation_eval.json @@ -0,0 +1,69 @@ +{ + "version": "2026-03-15", + "cases": [ + { + "id": "gen-001", + "type": "recommendation", + "request": { + "ingredients": ["chicken", "garlic", "lemon"], + "dietary_notes": "low carb", + "top_k": 2, + "filters": { "dietary_tags": ["low_carb"] }, + "include_diagnostics": true + }, + "expect_fallback": false, + "expect_warning": false + }, + { + "id": "gen-002", + "type": "recommendation", + "request": { + "ingredients": ["pasta", "garlic"], + "dietary_notes": "no dairy", + "top_k": 2, + "include_diagnostics": true + }, + "expect_fallback": false, + "expect_warning": true + }, + { + "id": "gen-003", + "type": "meal_plan", + "request": { + "dietary_profile": "high protein", + "calorie_target": 1800, + "days": 2, + "meals_per_day": 2, + "filters": { "dietary_tags": ["high_protein"] }, + "include_diagnostics": true + }, + "expect_fallback": false, + "expect_warning": false + }, + { + "id": "gen-004", + "type": "nutrition_qa", + "request": { + "question": "Which recipes are high in protein?", + "top_k": 3, + "filters": { "dietary_tags": ["high_protein"] }, + "include_diagnostics": true + }, + "expect_fallback": false, + "expect_warning": false + }, + { + "id": "gen-005", + "type": "meal_plan", + "request": { + "dietary_profile": "keto", + "days": 2, + "meals_per_day": 2, + "filters": { "dietary_tags": ["keto"] }, + "include_diagnostics": true + }, + "expect_fallback": true, + "expect_warning": true + } + ] +} diff --git a/eval/datasets/retrieval_eval.json b/eval/datasets/retrieval_eval.json new file mode 100644 index 0000000..c4b30d0 --- /dev/null +++ b/eval/datasets/retrieval_eval.json @@ -0,0 +1,46 @@ +{ + "version": "2026-03-15", + "cases": [ + { + "id": "retr-001", + "query": "chicken garlic lemon", + "top_k": 5, + "expected_relevant_ids": ["rec-002"], + "filters": null, + "excluded_ids": [] + }, + { + "id": "retr-002", + "query": "pasta primavera", + "top_k": 5, + "expected_relevant_ids": ["rec-001"], + "filters": { "cuisine": ["italian"] }, + "excluded_ids": ["rec-003", "rec-005"] + }, + { + "id": "retr-003", + "query": "vegan dinner high fiber", + "top_k": 5, + "expected_relevant_ids": ["rec-003", "rec-005", "rec-009"], + "filters": { "dietary_tags": ["vegan"] }, + "excluded_ids": ["rec-002", "rec-004", "rec-007"] + }, + { + "id": "retr-004", + "query": "quick breakfast", + "top_k": 5, + "expected_relevant_ids": ["rec-004", "rec-007", "rec-010"], + "filters": { "meal_type": ["breakfast"], "max_total_time_minutes": 15 }, + "excluded_ids": ["rec-002", "rec-003"] + }, + { + "id": "retr-005", + "query": "keto lunch", + "top_k": 5, + "expected_relevant_ids": [], + "filters": { "dietary_tags": ["keto"] }, + "excluded_ids": ["rec-001", "rec-002", "rec-003", "rec-004", "rec-005", "rec-006", "rec-007", "rec-008", "rec-009", "rec-010"], + "expect_empty": true + } + ] +} diff --git a/eval/metrics/generation_metrics.py b/eval/metrics/generation_metrics.py new file mode 100644 index 0000000..6622062 --- /dev/null +++ b/eval/metrics/generation_metrics.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from typing import Any, Dict, Iterable, List, Optional + +from backend.app.schemas.ai import ( + GroundedRecommendationResponse, + MealPlanResponse, + NutritionQAResponse, +) +from backend.app.schemas.recipe import RecipeFilters +from backend.app.services.rag_service import RagService + + +def extract_response_ids(response: Any, response_type: str) -> List[str]: + if response_type == "recommendation" and isinstance(response, GroundedRecommendationResponse): + return [item.recipe_id for item in response.recommended_recipes] + if response_type == "meal_plan" and isinstance(response, MealPlanResponse): + ids: List[str] = [] + for day in response.days: + ids.extend([meal.recipe_id for meal in day.meals]) + return ids + if response_type == "nutrition_qa" and isinstance(response, NutritionQAResponse): + return [source.id for source in response.supporting_sources] + return [] + + +def extract_warning_messages(response: Any, response_type: str) -> List[str]: + warnings: List[str] = [] + if hasattr(response, "warnings") and response.warnings: + warnings.extend(list(response.warnings)) + + if response_type == "recommendation" and isinstance(response, GroundedRecommendationResponse): + for item in response.recommended_recipes: + warnings.extend(item.warnings) + + if response_type == "meal_plan" and isinstance(response, MealPlanResponse): + for day in response.days: + for meal in day.meals: + warnings.extend(meal.warnings) + + return [msg for msg in warnings if msg] + + +def is_fallback_response(response: Any, response_type: str) -> bool: + if response_type == "recommendation" and isinstance(response, GroundedRecommendationResponse): + return len(response.recommended_recipes) == 0 + if response_type == "meal_plan" and isinstance(response, MealPlanResponse): + if not response.days: + return True + return all(len(day.meals) == 0 for day in response.days) + if response_type == "nutrition_qa" and isinstance(response, NutritionQAResponse): + return len(response.supporting_sources) == 0 + return False + + +def count_source_id_violations(response_ids: Iterable[str], retrieved_ids: Iterable[str]) -> int: + retrieved_set = set(retrieved_ids) + return len([rid for rid in response_ids if rid not in retrieved_set]) + + +def count_unknown_refs(response_ids: Iterable[str], recipe_map: Dict[str, Dict[str, Any]]) -> int: + return len([rid for rid in response_ids if rid not in recipe_map]) + + +def count_filter_violations( + response_ids: Iterable[str], + recipe_map: Dict[str, Dict[str, Any]], + filters: Optional[RecipeFilters], + rag: RagService, +) -> int: + if not filters: + return 0 + violations = 0 + for recipe_id in response_ids: + recipe = recipe_map.get(recipe_id) + if recipe and not rag._matches_filters(recipe, filters): + violations += 1 + return violations + + +def evaluate_generation_case( + response_type: str, + response: Any, + retrieved_ids: List[str], + recipe_map: Dict[str, Dict[str, Any]], + filters: Optional[RecipeFilters], + rag: RagService, + expect_fallback: bool, + expect_warning: bool, +) -> Dict[str, Any]: + response_ids = extract_response_ids(response, response_type) + warnings = extract_warning_messages(response, response_type) + fallback_detected = is_fallback_response(response, response_type) + + return { + "response_count": len(response_ids), + "retrieved_count": len(retrieved_ids), + "source_id_violations": count_source_id_violations(response_ids, retrieved_ids), + "unknown_recipe_refs": count_unknown_refs(response_ids, recipe_map), + "filter_violations": count_filter_violations(response_ids, recipe_map, filters, rag), + "warnings_present": bool(warnings), + "warning_expected": expect_warning, + "warning_correct": bool(warnings) == bool(expect_warning), + "fallback_detected": fallback_detected, + "fallback_expected": expect_fallback, + "fallback_correct": fallback_detected == bool(expect_fallback), + } diff --git a/eval/metrics/retrieval_metrics.py b/eval/metrics/retrieval_metrics.py new file mode 100644 index 0000000..5c4fc6a --- /dev/null +++ b/eval/metrics/retrieval_metrics.py @@ -0,0 +1,44 @@ +from typing import Any, Dict, List, Optional + +from backend.app.schemas.recipe import RecipeFilters +from backend.app.services.rag_service import RagService + + +def hit_at_k(retrieved_ids: List[str], expected_ids: List[str]) -> int: + return int(bool(set(retrieved_ids).intersection(expected_ids))) + + +def recall_at_k(retrieved_ids: List[str], expected_ids: List[str]) -> Optional[float]: + if not expected_ids: + return None + return len(set(retrieved_ids).intersection(expected_ids)) / float(len(set(expected_ids))) + + +def filter_violations(results: List[Dict[str, Any]], filters: Optional[RecipeFilters], rag: RagService) -> int: + if not filters: + return 0 + violations = 0 + for recipe in results: + if not rag._matches_filters(recipe, filters): + violations += 1 + return violations + + +def excluded_violations(retrieved_ids: List[str], excluded_ids: List[str]) -> int: + return len(set(retrieved_ids).intersection(excluded_ids)) + + +def evaluate_case( + rag: RagService, + results: List[Dict[str, Any]], + expected_ids: List[str], + excluded_ids: List[str], + filters: Optional[RecipeFilters], +) -> Dict[str, Any]: + retrieved_ids = [str(r.get("id", "")) for r in results] + return { + "hit_at_k": hit_at_k(retrieved_ids, expected_ids), + "recall_at_k": recall_at_k(retrieved_ids, expected_ids), + "filter_violations": filter_violations(results, filters, rag), + "excluded_violations": excluded_violations(retrieved_ids, excluded_ids), + } diff --git a/eval/runners/run_generation_eval.py b/eval/runners/run_generation_eval.py new file mode 100644 index 0000000..34419a7 --- /dev/null +++ b/eval/runners/run_generation_eval.py @@ -0,0 +1,345 @@ +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import os +import re +import subprocess +from typing import Any, Dict, List, Optional, Sequence, Type + +from backend.app.core.config import settings +from backend.app.schemas.ai import ( + GroundedRecommendationRequest, + GroundedRecommendationResponse, + LLMMealPlanPayload, + LLMNutritionQAPayload, + LLMRecommendationPayload, + MacroSummary, + MealPlanDay, + MealPlanMeal, + MealPlanRequest, + MealPlanResponse, + NutritionQARequest, + NutritionQAResponse, + RecommendedRecipe, +) +from backend.app.schemas.recipe import RecipeFilters +from backend.app.services.grounded_ai_service import GroundedAIService +from backend.app.services.llm_service import LLMService +from backend.app.services.rag_service import RagService +from eval.metrics.generation_metrics import evaluate_generation_case + + +class OfflineLLMService: + def generate_structured(self, system_prompt: str, user_prompt: str, schema_model: Type[Any]) -> Any: + context = _extract_context(user_prompt) + if schema_model is LLMRecommendationPayload: + return _offline_recommendations(context, user_prompt) + if schema_model is LLMMealPlanPayload: + return _offline_meal_plan(context, user_prompt) + if schema_model is LLMNutritionQAPayload: + return _offline_nutrition_qa(context) + raise RuntimeError(f"Unsupported schema model: {schema_model}") + + +def _extract_context(user_prompt: str) -> List[Dict[str, Any]]: + marker = "Recipes:\n" + if marker not in user_prompt: + return [] + payload = user_prompt.split(marker, 1)[1].strip() + if not payload: + return [] + try: + data = json.loads(payload) + if isinstance(data, list): + return data + except json.JSONDecodeError: + return [] + return [] + + +def _extract_int(label: str, text: str, default: int) -> int: + match = re.search(rf"{re.escape(label)}\s*(\d+)", text) + if match: + return int(match.group(1)) + return default + + +def _extract_dietary_profile(text: str) -> Optional[str]: + match = re.search(r"Dietary profile:\s*(.+)", text) + if match: + value = match.group(1).strip() + return value if value and value.lower() != "none" else None + return None + + +def _macro_summary(recipe: Dict[str, Any]) -> Optional[MacroSummary]: + calories = recipe.get("calories") + protein_g = recipe.get("protein_g") + carbs_g = recipe.get("carbs_g") + fat_g = recipe.get("fat_g") + if calories is None and protein_g is None and carbs_g is None and fat_g is None: + return None + return MacroSummary( + calories=calories, + protein_g=protein_g, + carbs_g=carbs_g, + fat_g=fat_g, + ) + + +def _offline_recommendations(context: Sequence[Dict[str, Any]], user_prompt: str) -> LLMRecommendationPayload: + items: List[RecommendedRecipe] = [] + for recipe in context[:3]: + items.append( + RecommendedRecipe( + recipe_id=str(recipe.get("id", "")), + title=recipe.get("title", ""), + why_selected="Matches the retrieved ingredients and dietary notes.", + substitutions=[], + warnings=[], + nutrition_summary=_macro_summary(recipe), + ) + ) + return LLMRecommendationPayload(recommended_recipes=items, warnings=[]) + + +def _offline_meal_plan(context: Sequence[Dict[str, Any]], user_prompt: str) -> LLMMealPlanPayload: + days = _extract_int("Days:", user_prompt, 1) + meals_per_day = _extract_int("Meals per day:", user_prompt, 3) + dietary_notes = _extract_dietary_profile(user_prompt) + + if not context: + return LLMMealPlanPayload(days=[], dietary_notes=dietary_notes, warnings=["No recipes provided."]) + + plan_days: List[MealPlanDay] = [] + for day_index in range(days): + meals: List[MealPlanMeal] = [] + for meal_index in range(meals_per_day): + recipe = context[(day_index * meals_per_day + meal_index) % len(context)] + meal_type = recipe.get("meal_type") + if isinstance(meal_type, list) and meal_type: + meal_type_value = str(meal_type[0]) + else: + meal_type_value = meal_type + meals.append( + MealPlanMeal( + recipe_id=str(recipe.get("id", "")), + title=recipe.get("title", ""), + meal_type=meal_type_value, + why_selected="Balanced fit for the requested plan.", + estimated_calories=recipe.get("calories"), + macros=_macro_summary(recipe), + warnings=[], + ) + ) + plan_days.append(MealPlanDay(day=day_index + 1, meals=meals, estimated_totals=None, notes=None)) + + return LLMMealPlanPayload(days=plan_days, dietary_notes=dietary_notes, warnings=[]) + + +def _offline_nutrition_qa(context: Sequence[Dict[str, Any]]) -> LLMNutritionQAPayload: + if not context: + return LLMNutritionQAPayload( + answer="No supporting recipes were provided.", + supporting_recipe_ids=[], + warnings=["No sources available."], + ) + top = context[:3] + titles = ", ".join(recipe.get("title", "") for recipe in top if recipe.get("title")) + answer = f"Based on the retrieved recipes, relevant options include: {titles}." + supporting_ids = [str(recipe.get("id", "")) for recipe in top] + return LLMNutritionQAPayload(answer=answer, supporting_recipe_ids=supporting_ids, warnings=[]) + + +def _load_dataset(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def _git_commit() -> Optional[str]: + try: + output = subprocess.check_output(["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL) + return output.decode("utf-8").strip() + except Exception: + return None + + +def _filters_from_request(request: Any) -> Optional[RecipeFilters]: + if hasattr(request, "filters"): + return request.filters + return None + + +def _retrieved_ids_from_response(response: Any) -> List[str]: + if isinstance(response, GroundedRecommendationResponse): + return [ref.id for ref in response.source_recipes] + if isinstance(response, MealPlanResponse): + return [ref.id for ref in response.source_recipes] + if isinstance(response, NutritionQAResponse): + if response.retrieval_diagnostics and response.retrieval_diagnostics.top_ids: + return list(response.retrieval_diagnostics.top_ids) + return [ref.id for ref in response.supporting_sources] + return [] + + +def _response_payload(response: Any) -> Dict[str, Any]: + if hasattr(response, "model_dump"): + return response.model_dump() + return {} + + +def run(args: argparse.Namespace) -> None: + dataset = _load_dataset(args.dataset) + cases = dataset.get("cases", []) + + if not os.path.exists(settings.FAISS_INDEX_PATH) or not os.path.exists(settings.RECIPES_PKL_PATH): + raise RuntimeError( + "RAG artifacts not found. Build them with: " + "python scripts/build_rag_index.py --recipes data/sample_recipes.json --out-dir models/" + ) + + rag = RagService(settings.FAISS_INDEX_PATH, settings.RECIPES_PKL_PATH) + if args.mode == "openai": + if not settings.OPENAI_API_KEY: + raise RuntimeError("OPENAI_API_KEY is required for --mode openai") + llm: Any = LLMService() + else: + llm = OfflineLLMService() + + grounded = GroundedAIService(rag=rag, llm=llm) + recipe_map = {str(recipe.get("id", "")): recipe for recipe in rag.recipes} + + results: List[Dict[str, Any]] = [] + for case in cases: + case_type = case.get("type") + request_payload = case.get("request", {}) + + if case_type == "recommendation": + request = GroundedRecommendationRequest.model_validate(request_payload) + response = grounded.grounded_recommendations(request) + elif case_type == "meal_plan": + request = MealPlanRequest.model_validate(request_payload) + response = grounded.generate_meal_plan(request) + elif case_type == "nutrition_qa": + request = NutritionQARequest.model_validate(request_payload) + response = grounded.answer_nutrition_question(request) + else: + raise ValueError(f"Unknown eval case type: {case_type}") + + retrieved_ids = _retrieved_ids_from_response(response) + metrics = evaluate_generation_case( + response_type=case_type, + response=response, + retrieved_ids=retrieved_ids, + recipe_map=recipe_map, + filters=_filters_from_request(request), + rag=rag, + expect_fallback=bool(case.get("expect_fallback", False)), + expect_warning=bool(case.get("expect_warning", False)), + ) + + results.append( + { + "case_id": case.get("id"), + "type": case_type, + "request": request_payload, + "response": _response_payload(response), + "retrieved_ids": retrieved_ids, + **metrics, + } + ) + + summary = { + "cases": len(results), + "fallback_correct_rate": _ratio([row["fallback_correct"] for row in results]), + "warning_correct_rate": _ratio([row["warning_correct"] for row in results]), + "source_id_violations": sum(row["source_id_violations"] for row in results), + "unknown_recipe_refs": sum(row["unknown_recipe_refs"] for row in results), + "filter_violations": sum(row["filter_violations"] for row in results), + } + + os.makedirs(args.out_dir, exist_ok=True) + run_id = args.run_id or dt.datetime.now(dt.timezone.utc).strftime("%Y%m%d_%H%M%S") + payload = { + "run_id": run_id, + "dataset_path": args.dataset, + "dataset_version": dataset.get("version"), + "mode": args.mode, + "faiss_index_path": settings.FAISS_INDEX_PATH, + "recipes_pkl_path": settings.RECIPES_PKL_PATH, + "git_commit": _git_commit(), + "summary": summary, + "cases": results, + } + + json_path = os.path.join(args.out_dir, f"generation_eval_{run_id}.json") + with open(json_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2) + + csv_path = os.path.join(args.out_dir, f"generation_eval_{run_id}.csv") + csv_fields = [ + "case_id", + "type", + "fallback_correct", + "warning_correct", + "source_id_violations", + "unknown_recipe_refs", + "filter_violations", + "response_count", + "retrieved_count", + ] + with open(csv_path, "w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=csv_fields) + writer.writeheader() + for row in results: + writer.writerow({field: row.get(field) for field in csv_fields}) + + print("Generation evaluation complete.") + print(f"Run ID: {run_id}") + print(f"JSON: {json_path}") + print(f"CSV: {csv_path}") + print( + "Summary: fallback_correct_rate={:.2f}, warning_correct_rate={:.2f}, " + "source_id_violations={}, filter_violations={}".format( + summary["fallback_correct_rate"], + summary["warning_correct_rate"], + summary["source_id_violations"], + summary["filter_violations"], + ) + ) + + +def _ratio(flags: List[bool]) -> float: + if not flags: + return 0.0 + return sum(1 for flag in flags if flag) / float(len(flags)) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run grounded generation evaluation for NourishAI.") + parser.add_argument( + "--dataset", + default="eval/datasets/grounded_generation_eval.json", + help="Path to grounded generation evaluation dataset JSON.", + ) + parser.add_argument( + "--out-dir", + default="eval/outputs", + help="Directory to write evaluation artifacts.", + ) + parser.add_argument( + "--mode", + choices=["offline", "openai"], + default="offline", + help="offline uses deterministic stub LLM; openai calls the real API.", + ) + parser.add_argument("--run-id", default=None, help="Optional run identifier.") + args = parser.parse_args() + run(args) + + +if __name__ == "__main__": + main() diff --git a/eval/runners/run_retrieval_eval.py b/eval/runners/run_retrieval_eval.py new file mode 100644 index 0000000..ae2a543 --- /dev/null +++ b/eval/runners/run_retrieval_eval.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import os +import subprocess +from typing import Any, Dict, List, Optional + +from backend.app.core.config import settings +from backend.app.schemas.recipe import RecipeFilters +from backend.app.services.rag_service import RagService +from eval.metrics.retrieval_metrics import evaluate_case + + +def _load_dataset(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def _build_filters(raw: Optional[Dict[str, Any]]) -> Optional[RecipeFilters]: + if raw is None: + return None + return RecipeFilters.model_validate(raw) + + +def _mean(values: List[float]) -> float: + if not values: + return 0.0 + return sum(values) / float(len(values)) + + +def _git_commit() -> Optional[str]: + try: + output = subprocess.check_output(["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL) + return output.decode("utf-8").strip() + except Exception: + return None + + +def _write_csv(path: str, rows: List[Dict[str, Any]], fieldnames: List[str]) -> None: + with open(path, "w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def _run_strategy( + rag: RagService, + case: Dict[str, Any], + strategy_name: str, + use_filters: bool, + rerank: bool, +) -> Dict[str, Any]: + filters = _build_filters(case.get("filters")) if use_filters else None + metrics_filters = _build_filters(case.get("filters")) + results, diagnostics = rag.search_with_diagnostics( + query=case["query"], + top_k=case.get("top_k", 5), + filters=filters, + rerank=rerank, + candidate_k=case.get("candidate_k"), + include_diagnostics=True, + ) + + metrics = evaluate_case( + rag=rag, + results=results, + expected_ids=case.get("expected_relevant_ids", []), + excluded_ids=case.get("excluded_ids", []), + filters=metrics_filters, + ) + + retrieved_ids = [r.get("id") for r in results] + retrieved_scores = [r.get("score") for r in results] + expect_empty = bool(case.get("expect_empty", False)) + empty_ok = (len(results) == 0) == expect_empty + + return { + "case_id": case.get("id"), + "strategy": strategy_name, + "query": case.get("query"), + "top_k": case.get("top_k", 5), + "filters": case.get("filters"), + "expected_relevant_ids": case.get("expected_relevant_ids", []), + "excluded_ids": case.get("excluded_ids", []), + "expect_empty": expect_empty, + "empty_ok": empty_ok, + "retrieved_ids": retrieved_ids, + "retrieved_scores": retrieved_scores, + "diagnostics": diagnostics.model_dump() if diagnostics else None, + **metrics, + } + + +def run(args: argparse.Namespace) -> None: + dataset = _load_dataset(args.dataset) + cases = dataset.get("cases", []) + if not os.path.exists(settings.FAISS_INDEX_PATH) or not os.path.exists(settings.RECIPES_PKL_PATH): + raise RuntimeError( + "RAG artifacts not found. Build them with: " + "python scripts/build_rag_index.py --recipes data/sample_recipes.json --out-dir models/" + ) + rag = RagService(settings.FAISS_INDEX_PATH, settings.RECIPES_PKL_PATH) + + strategies = [ + ("baseline", False, False), + ("filtered", True, False), + ("reranked", True, True), + ] + + results: List[Dict[str, Any]] = [] + for case in cases: + for strategy_name, use_filters, rerank in strategies: + results.append(_run_strategy(rag, case, strategy_name, use_filters, rerank)) + + summary: Dict[str, Any] = {} + for strategy_name, _, _ in strategies: + strategy_rows = [row for row in results if row["strategy"] == strategy_name] + hit_rates = [row["hit_at_k"] for row in strategy_rows] + recalls = [row["recall_at_k"] for row in strategy_rows if row["recall_at_k"] is not None] + summary[strategy_name] = { + "cases": len(strategy_rows), + "hit_rate": _mean(hit_rates), + "recall_avg": _mean(recalls), + "filter_violations": sum(row["filter_violations"] for row in strategy_rows), + "excluded_violations": sum(row["excluded_violations"] for row in strategy_rows), + "empty_correct": sum(1 for row in strategy_rows if row["empty_ok"]), + } + + os.makedirs(args.out_dir, exist_ok=True) + run_id = args.run_id or dt.datetime.now(dt.timezone.utc).strftime("%Y%m%d_%H%M%S") + + payload = { + "run_id": run_id, + "dataset_path": args.dataset, + "dataset_version": dataset.get("version"), + "faiss_index_path": settings.FAISS_INDEX_PATH, + "recipes_pkl_path": settings.RECIPES_PKL_PATH, + "git_commit": _git_commit(), + "strategies": [name for name, _, _ in strategies], + "summary": summary, + "cases": results, + } + + json_path = os.path.join(args.out_dir, f"retrieval_eval_{run_id}.json") + with open(json_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2) + + csv_path = os.path.join(args.out_dir, f"retrieval_eval_{run_id}.csv") + csv_fields = [ + "case_id", + "strategy", + "query", + "top_k", + "hit_at_k", + "recall_at_k", + "filter_violations", + "excluded_violations", + "expect_empty", + "empty_ok", + "retrieved_ids", + "retrieved_scores", + ] + csv_rows = [ + {field: row.get(field) for field in csv_fields} + for row in results + ] + _write_csv(csv_path, csv_rows, csv_fields) + + print("Retrieval evaluation complete.") + print(f"Run ID: {run_id}") + print(f"JSON: {json_path}") + print(f"CSV: {csv_path}") + print("Summary:") + for strategy_name, metrics in summary.items(): + print(f"- {strategy_name}: hit_rate={metrics['hit_rate']:.2f}, recall_avg={metrics['recall_avg']:.2f}, " + f"filter_violations={metrics['filter_violations']}, excluded_violations={metrics['excluded_violations']}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run retrieval evaluation for NourishAI.") + parser.add_argument( + "--dataset", + default="eval/datasets/retrieval_eval.json", + help="Path to retrieval evaluation dataset JSON.", + ) + parser.add_argument( + "--out-dir", + default="eval/outputs", + help="Directory to write evaluation artifacts.", + ) + parser.add_argument("--run-id", default=None, help="Optional run identifier.") + args = parser.parse_args() + run(args) + + +if __name__ == "__main__": + main() diff --git a/models/faiss.index b/models/faiss.index deleted file mode 100644 index e69de29..0000000 diff --git a/models/recipe.pkl b/models/recipe.pkl deleted file mode 100644 index e69de29..0000000 diff --git a/requirements.txt b/requirements.txt index 2c1a81d..c09f1e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -79,5 +79,7 @@ pydantic-settings gdown faiss-cpu sentence-transformers +qdrant-client +rank-bm25 torch streamlit-lottie diff --git a/scripts/build_rag_index.py b/scripts/build_rag_index.py new file mode 100644 index 0000000..c804277 --- /dev/null +++ b/scripts/build_rag_index.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import argparse +import json +import os +import pickle +import re +from typing import Any, Dict, List + +import faiss +import numpy as np +from sentence_transformers import SentenceTransformer + + +def load_recipes(path: str) -> List[Dict[str, Any]]: + with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle) + if not isinstance(data, list): + raise ValueError("Recipe JSON must be a list of objects") + + recipes: List[Dict[str, Any]] = [] + for item in data: + if not isinstance(item, dict): + continue + recipes.append( + { + "id": str(item.get("id", "")), + "title": str(item.get("title", "")), + "ingredients": _as_list(item.get("ingredients")), + "instructions": _as_list(item.get("instructions")), + "calories": item.get("calories"), + "protein_g": item.get("protein_g"), + "carbs_g": item.get("carbs_g"), + "fat_g": item.get("fat_g"), + "cuisine": _as_list(item.get("cuisine")), + "meal_type": _as_list(item.get("meal_type")), + "dietary_tags": _as_list(item.get("dietary_tags")), + "allergens": _as_list(item.get("allergens")), + "prep_time_minutes": item.get("prep_time_minutes"), + "cook_time_minutes": item.get("cook_time_minutes"), + "total_time_minutes": item.get("total_time_minutes"), + } + ) + return recipes + + +def build_corpus(recipes: List[Dict[str, Any]]) -> List[str]: + texts: List[str] = [] + for recipe in recipes: + title = recipe.get("title") or "" + ingredients = recipe.get("ingredients") or [] + normalized = [normalize_ingredient(str(i)) for i in ingredients if str(i).strip()] + ingredient_text = ", ".join(normalized) if normalized else ", ".join(str(i) for i in ingredients) + texts.append(f"{title}. Ingredients: {ingredient_text}") + return texts + + +def build_index(texts: List[str], model: SentenceTransformer) -> np.ndarray: + embeddings = model.encode(texts, show_progress_bar=True) + vectors = np.asarray(embeddings, dtype="float32") + faiss.normalize_L2(vectors) + return vectors + + +def normalize_ingredient(text: str) -> str: + cleaned = text.lower().strip() + cleaned = re.sub(r"\(.*?\)", "", cleaned) + cleaned = re.sub(r"[^a-z0-9\s-]", "", cleaned) + cleaned = cleaned.replace("-", " ") + cleaned = re.sub(r"\s+", " ", cleaned).strip() + + cleaned = _singularize(cleaned) + aliases = { + "garbanzo beans": "chickpea", + "garbanzo bean": "chickpea", + "scallions": "green onion", + "spring onions": "green onion", + "courgette": "zucchini", + } + return aliases.get(cleaned, cleaned) + + +def _singularize(text: str) -> str: + if text.endswith("ies") and len(text) > 3: + return text[:-3] + "y" + if text.endswith("es") and len(text) > 3: + return text[:-2] + if text.endswith("s") and len(text) > 3: + return text[:-1] + return text + + +def _as_list(value: Any) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v) for v in value if str(v).strip()] + if isinstance(value, str): + if "," in value: + return [v.strip() for v in value.split(",") if v.strip()] + return [value.strip()] + return [str(value)] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build FAISS index for NourishAI recipes") + parser.add_argument("--recipes", required=True, help="Path to recipes JSON file") + parser.add_argument("--out-dir", required=True, help="Output directory for faiss.index and recipe.pkl") + args = parser.parse_args() + + recipes = load_recipes(args.recipes) + if not recipes: + raise ValueError("No recipes found in input JSON") + + model = SentenceTransformer("all-MiniLM-L6-v2") + texts = build_corpus(recipes) + vectors = build_index(texts, model) + + index = faiss.IndexFlatIP(vectors.shape[1]) + index.add(vectors) + + os.makedirs(args.out_dir, exist_ok=True) + index_path = os.path.join(args.out_dir, "faiss.index") + recipes_path = os.path.join(args.out_dir, "recipe.pkl") + + faiss.write_index(index, index_path) + with open(recipes_path, "wb") as handle: + pickle.dump(recipes, handle) + + print(f"Indexed {len(recipes)} recipes") + print(f"FAISS index: {index_path}") + print(f"Recipe data: {recipes_path}") + + +if __name__ == "__main__": + main()